/*
 * G R E E N    H I L L S    S O F T W A R E
 * Copyright (C) 2015 by Green Hills Software, Inc.
 *
 * This software is proprietary and is the property of Green Hills
 * Software, Inc.  Software is furnished under a license agreement and
 * is not to be disclosed to others except by the written permission
 * of the President of Green Hills Software, Inc.
 * -------------------------------------------------------------------
 */

/*
 * Implement atomic operations used by C++11 atomic headers.  Some targets
 * have instructions allowing atomic operations, while others require locks.
 *
 *       This file is not used by ARM, ARM64, Power, or x86.
 */
#ifndef __ATOMIC_GHS_H
#define __ATOMIC_GHS_H
#include <yvals.h>
#include <xatomic.h>
#include <ghs_barrier.h>
#include <ghs_atomic.h>

#ifdef __cplusplus
extern "C" {
#endif /* __cplusplus */

// The position of the barrier with respect to the atomic access
#define _POS_PRE 0	// Before the access
#define _POS_POST 1	// After the access
#define _POS_FENCE 2	// A fence
// The kind of atomic access
#define _LDST_LOAD 0    // A Load only
#define _LDST_STORE 1   // A Store only
#define _LDST_RMWLOOP 2	// A RMW with that requires the load to complete
#define _LDST_OTHER 3   // Other

#if defined(__DO_DEF)
/* atomic_ghs.c uses this */
#define __INLINE extern
#else
#define __INLINE extern inline __attribute__((always_inline))
#endif

#if !defined(__OUT_OF_LINE_ATOMICS)

#if defined(_NOBARRIERS)
#undef _GHS_ATOMIC_BARRIER
#define __GHS_ATOMIC_FENCE(POS, LDST, MO, DAT) _GHS_ATOMIC_NOP()
extern inline _GHS_ATOMIC_NOP(void) {}
#elif !defined(_HAS_ATOMIC_INTRINSIC)
#define __GHS_ATOMIC_FENCE(POS, LDST, MO, DAT) _GHS_ATOMIC_BARRIER()
#endif /* _NOBARRIERS / _HAS_ATOMIC_INTRINSIC */

_STD_BEGIN
#if (_ATOMIC_MAXBYTES_LOCK_FREE >= 1) && (_ATOMIC_MINBYTES_LOCK_FREE <= 1)

__INLINE
void _Atomic_store_1(volatile _Uint1_t *_Tgt,
	_Uint1_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_STORE, _Order, 0);
#ifdef _ATOMIC_FENCE_USES_EXCHANGE
    _Uint1_t prev = 0;
    do {
	prev = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_1(_Tgt, prev, _Value));
#else /* _ATOMIC_FENCE_USES_EXCHANGE */
    *(volatile __nosideeffect _Uint1_t *)_Tgt = _Value;
#endif /* _ATOMIC_FENCE_USES_EXCHANGE */
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_STORE, _Order, 0);
}

__INLINE
_Uint1_t _Atomic_load_1(volatile _Uint1_t *_Tgt, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_LOAD, _Order, 0);
#ifdef _ATOMIC_FENCE_USES_EXCHANGE
    _Uint1_t ret = *_Tgt;
    do {
	ret = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_1(_Tgt, ret, ret));
#else /* _ATOMIC_FENCE_USES_EXCHANGE */
    _Uint1_t ret = *_Tgt;
#endif /* _ATOMIC_FENCE_USES_EXCHANGE */
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_LOAD, _Order, ret);
    return ret;
}

__INLINE
_Uint1_t _Atomic_exchange_1(volatile _Uint1_t *_Tgt,
	_Uint1_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint1_t ret = 0;
    do {
	ret = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_1(_Tgt, ret, _Value));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_strong_1(volatile _Uint1_t *_Tgt,
	_Uint1_t *_Exp, _Uint1_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint1_t oldval = *_Tgt, newval = *_Exp;
    int ret = 0;
    if ((*_Tgt == newval) && !_GHS_COMPARE_EXCHANGE_1(_Tgt, newval, _Value)) {
	ret = 1;
    } else {
	*_Exp = oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_weak_1(volatile _Uint1_t *_Tgt,
	_Uint1_t *_Exp, _Uint1_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint1_t oldval = *_Tgt, newval = *_Exp;
    int ret = 0;
    if ((*_Tgt == newval) && !_GHS_COMPARE_EXCHANGE_1(_Tgt, newval, _Value)) {
	ret = 1;
    } else {
	*_Exp = oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
_Uint1_t _Atomic_fetch_add_1(volatile _Uint1_t *_Tgt,
	_Uint1_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint1_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret + _Value;
    } while (_GHS_COMPARE_EXCHANGE_1(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint1_t _Atomic_fetch_sub_1(volatile _Uint1_t *_Tgt,
	_Uint1_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint1_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret - _Value;
    } while (_GHS_COMPARE_EXCHANGE_1(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint1_t _Atomic_fetch_and_1(volatile _Uint1_t *_Tgt,
	_Uint1_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint1_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret & _Value;
    } while (_GHS_COMPARE_EXCHANGE_1(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint1_t _Atomic_fetch_or_1(volatile _Uint1_t *_Tgt,
	_Uint1_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint1_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret | _Value;
    } while (_GHS_COMPARE_EXCHANGE_1(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint1_t _Atomic_fetch_xor_1(volatile _Uint1_t *_Tgt,
	_Uint1_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint1_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret ^ _Value;
    } while (_GHS_COMPARE_EXCHANGE_1(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

#endif /*(_ATOMIC_MAXBYTES_LOCK_FREE>=1) && (_ATOMIC_MINBYTES_LOCK_FREE<=1)*/


#if (_ATOMIC_MAXBYTES_LOCK_FREE >= 2) && (_ATOMIC_MINBYTES_LOCK_FREE <= 2)

__INLINE
void _Atomic_store_2(volatile _Uint2_t *_Tgt,
	_Uint2_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_STORE, _Order, 0);
#ifdef _ATOMIC_FENCE_USES_EXCHANGE
    _Uint2_t prev = 0;
    do {
	prev = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_2(_Tgt, prev, _Value));
#else /* _ATOMIC_FENCE_USES_EXCHANGE */
    *(volatile __nosideeffect _Uint2_t *)_Tgt = _Value;
#endif /* _ATOMIC_FENCE_USES_EXCHANGE */
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_STORE, _Order, 0);
}

__INLINE
_Uint2_t _Atomic_load_2(volatile _Uint2_t *_Tgt, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_LOAD, _Order, 0);
#ifdef _ATOMIC_FENCE_USES_EXCHANGE
    _Uint2_t ret = *_Tgt;
    do {
	ret = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_2(_Tgt, ret, ret));
#else /* _ATOMIC_FENCE_USES_EXCHANGE */
    _Uint2_t ret = *_Tgt;
#endif /* _ATOMIC_FENCE_USES_EXCHANGE */
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_LOAD, _Order, ret);
    return ret;
}

__INLINE
_Uint2_t _Atomic_exchange_2(volatile _Uint2_t *_Tgt,
	_Uint2_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint2_t ret = 0;
    do {
	ret = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_2(_Tgt, ret, _Value));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_strong_2(volatile _Uint2_t *_Tgt,
	_Uint2_t *_Exp, _Uint2_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint2_t oldval = *_Tgt, newval = *_Exp;
    int ret = 0;
    if ((*_Tgt == newval) && !_GHS_COMPARE_EXCHANGE_2(_Tgt, newval, _Value)) {
	ret = 1;
    } else {
	*_Exp = oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_weak_2(volatile _Uint2_t *_Tgt,
	_Uint2_t *_Exp, _Uint2_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint2_t oldval = *_Tgt, newval = *_Exp;
    int ret = 0;
    if ((*_Tgt == newval) && !_GHS_COMPARE_EXCHANGE_2(_Tgt, newval, _Value)) {
	ret = 1;
    } else {
	*_Exp = oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
_Uint2_t _Atomic_fetch_add_2(volatile _Uint2_t *_Tgt,
	_Uint2_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint2_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret + _Value;
    } while (_GHS_COMPARE_EXCHANGE_2(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint2_t _Atomic_fetch_sub_2(volatile _Uint2_t *_Tgt,
	_Uint2_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint2_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret - _Value;
    } while (_GHS_COMPARE_EXCHANGE_2(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint2_t _Atomic_fetch_and_2(volatile _Uint2_t *_Tgt,
	_Uint2_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint2_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret & _Value;
    } while (_GHS_COMPARE_EXCHANGE_2(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint2_t _Atomic_fetch_or_2(volatile _Uint2_t *_Tgt,
	_Uint2_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint2_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret | _Value;
    } while (_GHS_COMPARE_EXCHANGE_2(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint2_t _Atomic_fetch_xor_2(volatile _Uint2_t *_Tgt,
	_Uint2_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint2_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret ^ _Value;
    } while (_GHS_COMPARE_EXCHANGE_2(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

#endif /*(_ATOMIC_MAXBYTES_LOCK_FREE>=2) && (_ATOMIC_MINBYTES_LOCK_FREE<=2)*/


#if (_ATOMIC_MAXBYTES_LOCK_FREE >= 4)

__INLINE
void _Atomic_store_4(volatile _Uint4_t *_Tgt,
	_Uint4_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_STORE, _Order, 0);
#ifdef _ATOMIC_FENCE_USES_EXCHANGE
    _Uint4_t prev = 0;
    do {
	prev = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_4(_Tgt, prev, _Value));
#else /* _ATOMIC_FENCE_USES_EXCHANGE */
    *(volatile __nosideeffect _Uint4_t *)_Tgt = _Value;
#endif /* _ATOMIC_FENCE_USES_EXCHANGE */
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_STORE, _Order, 0);
}

__INLINE
_Uint4_t _Atomic_load_4(volatile _Uint4_t *_Tgt, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_LOAD, _Order, 0);
#ifdef _ATOMIC_FENCE_USES_EXCHANGE
    _Uint4_t ret = *_Tgt;
    do {
	ret = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_4(_Tgt, ret, ret));
#else /* _ATOMIC_FENCE_USES_EXCHANGE */
    _Uint4_t ret = *_Tgt;
#endif /* _ATOMIC_FENCE_USES_EXCHANGE */
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_LOAD, _Order, ret);
    return ret;
}

__INLINE
_Uint4_t _Atomic_exchange_4(volatile _Uint4_t *_Tgt,
	_Uint4_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint4_t ret = 0;
    do {
	ret = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_4(_Tgt, ret, _Value));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_strong_4(volatile _Uint4_t *_Tgt,
	_Uint4_t *_Exp, _Uint4_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint4_t oldval = *_Tgt, newval = *_Exp;
    int ret = 0;
    if ((*_Tgt == newval) && !_GHS_COMPARE_EXCHANGE_4(_Tgt, newval, _Value)) {
	ret = 1;
    } else {
	*_Exp = oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_weak_4(volatile _Uint4_t *_Tgt,
	_Uint4_t *_Exp, _Uint4_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint4_t oldval = *_Tgt, newval = *_Exp;
    int ret = 0;
    if ((*_Tgt == newval) && !_GHS_COMPARE_EXCHANGE_4(_Tgt, newval, _Value)) {
	ret = 1;
    } else {
	*_Exp = oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

#if (_ATOMIC_MINBYTES_LOCK_FREE == 4)
__INLINE
int _Atomic_compare_exchange_strong_4_1(volatile _Uint4_t *_Tgt,
   	_Uint1_t *_Exp, _Uint1_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint4_t oldval = *_Tgt;
    _Uint1_t newval = *_Exp;
    int ret = 0;
    if (/*(*_Tgt == newval) &&*/ !_GHS_COMPARE_EXCHANGE_4_1(_Tgt, (_Uint4_t)newval, (_Uint4_t)_Value)) {
	ret = 1;
    } else {
	*_Exp = (_Uint1_t)oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_strong_4_2(volatile _Uint4_t *_Tgt,
   	_Uint2_t *_Exp, _Uint2_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint4_t oldval = *_Tgt;
    _Uint2_t newval = *_Exp;
    int ret = 0;
    if (/*(*_Tgt == newval) &&*/ !_GHS_COMPARE_EXCHANGE_4_2(_Tgt, (_Uint4_t)newval, (_Uint4_t)_Value)) {
	ret = 1;
    } else {
	*_Exp = (_Uint2_t)oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_weak_4_1(volatile _Uint4_t *_Tgt,
   	_Uint1_t *_Exp, _Uint1_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint4_t oldval = *_Tgt;
    _Uint1_t newval = *_Exp;
    int ret = 0;
    if (/*(*_Tgt == newval) &&*/ !_GHS_COMPARE_EXCHANGE_4_1(_Tgt, (_Uint4_t)newval, (_Uint4_t)_Value)) {
	ret = 1;
    } else {
	*_Exp = (_Uint1_t)oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_weak_4_2(volatile _Uint4_t *_Tgt,
   	_Uint2_t *_Exp, _Uint2_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint4_t oldval = *_Tgt;
    _Uint2_t newval = *_Exp;
    int ret = 0;
    if (/*(*_Tgt == newval) && */!_GHS_COMPARE_EXCHANGE_4_2(_Tgt, (_Uint4_t)newval, (_Uint4_t)_Value)) {
	ret = 1;
    } else {
	*_Exp = (_Uint2_t)oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}


#endif	/* _ATOMIC_MINBYTES_LOCK_FREE == 4 */

__INLINE
_Uint4_t _Atomic_fetch_add_4(volatile _Uint4_t *_Tgt,
	_Uint4_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint4_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret + _Value;
    } while (_GHS_COMPARE_EXCHANGE_4(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint4_t _Atomic_fetch_sub_4(volatile _Uint4_t *_Tgt,
	_Uint4_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint4_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret - _Value;
    } while (_GHS_COMPARE_EXCHANGE_4(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint4_t _Atomic_fetch_and_4(volatile _Uint4_t *_Tgt,
	_Uint4_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint4_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret & _Value;
    } while (_GHS_COMPARE_EXCHANGE_4(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint4_t _Atomic_fetch_or_4(volatile _Uint4_t *_Tgt,
	_Uint4_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint4_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret | _Value;
    } while (_GHS_COMPARE_EXCHANGE_4(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint4_t _Atomic_fetch_xor_4(volatile _Uint4_t *_Tgt,
	_Uint4_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint4_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret ^ _Value;
    } while (_GHS_COMPARE_EXCHANGE_4(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

#endif /* (_ATOMIC_MAXBYTES_LOCK_FREE >= 4) */


#if (_ATOMIC_MAXBYTES_LOCK_FREE >= 8)

__INLINE
void _Atomic_store_8(volatile _Uint8_t *_Tgt,
	_Uint8_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_STORE, _Order, 0);
#if defined(_ATOMIC_FENCE_USES_EXCHANGE) || defined(_ATOMIC8_FENCE_USES_EXCHANGE)
    _Uint8_t prev = 0;
    do {
	prev = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_8(_Tgt, prev, _Value));
#else /* _ATOMIC_FENCE_USES_EXCHANGE */
    *(volatile __nosideeffect _Uint8_t *)_Tgt = _Value;
#endif /* _ATOMIC_FENCE_USES_EXCHANGE */
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_STORE, _Order, 0);
}

__INLINE
_Uint8_t _Atomic_load_8(volatile _Uint8_t *_Tgt, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_LOAD, _Order, 0);
#if defined(_ATOMIC_FENCE_USES_EXCHANGE) || defined(_ATOMIC8_FENCE_USES_EXCHANGE)
    _Uint8_t ret = *_Tgt;
    do {
	ret = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_8(_Tgt, ret, ret));
#else /* _ATOMIC_FENCE_USES_EXCHANGE */
    _Uint8_t ret = *_Tgt;
#endif /* _ATOMIC_FENCE_USES_EXCHANGE */
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_LOAD, _Order, ret);
    return ret;
}

__INLINE
_Uint8_t _Atomic_exchange_8(volatile _Uint8_t *_Tgt,
	_Uint8_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint8_t ret = 0;
    do {
	ret = *_Tgt;
    } while (_GHS_COMPARE_EXCHANGE_8(_Tgt, ret, _Value));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_strong_8(volatile _Uint8_t *_Tgt,
	_Uint8_t *_Exp, _Uint8_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint8_t oldval = *_Tgt, newval = *_Exp;
    int ret = 0;
    if ((*_Tgt == newval) && !_GHS_COMPARE_EXCHANGE_8(_Tgt, newval, _Value)) {
	ret = 1;
    } else {
	*_Exp = oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
int _Atomic_compare_exchange_weak_8(volatile _Uint8_t *_Tgt,
	_Uint8_t *_Exp, _Uint8_t _Value,
	memory_order _Order1, memory_order _Order2)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    _Uint8_t oldval = *_Tgt, newval = *_Exp;
    int ret = 0;
    if ((*_Tgt == newval) && !_GHS_COMPARE_EXCHANGE_8(_Tgt, newval, _Value)) {
	ret = 1;
    } else {
	*_Exp = oldval;
    }
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_OTHER, _Order1 > _Order2 ? _Order1 : _Order2, 0);
    return ret;
}

__INLINE
_Uint8_t _Atomic_fetch_add_8(volatile _Uint8_t *_Tgt,
	_Uint8_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint8_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret + _Value;
    } while (_GHS_COMPARE_EXCHANGE_8(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint8_t _Atomic_fetch_sub_8(volatile _Uint8_t *_Tgt,
	_Uint8_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint8_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret - _Value;
    } while (_GHS_COMPARE_EXCHANGE_8(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint8_t _Atomic_fetch_and_8(volatile _Uint8_t *_Tgt,
	_Uint8_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint8_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret & _Value;
    } while (_GHS_COMPARE_EXCHANGE_8(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint8_t _Atomic_fetch_or_8(volatile _Uint8_t *_Tgt,
	_Uint8_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint8_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret | _Value;
    } while (_GHS_COMPARE_EXCHANGE_8(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

__INLINE
_Uint8_t _Atomic_fetch_xor_8(volatile _Uint8_t *_Tgt,
	_Uint8_t _Value, memory_order _Order)
{
    __GHS_ATOMIC_FENCE(_POS_PRE, _LDST_RMWLOOP, _Order, 0);
    _Uint8_t ret = 0, newval = 0;
    do {
	ret = *_Tgt;
	newval = ret ^ _Value;
    } while (_GHS_COMPARE_EXCHANGE_8(_Tgt, ret, newval));
    __GHS_ATOMIC_FENCE(_POS_POST, _LDST_RMWLOOP, _Order, ret);
    return ret;
}

#endif /* (_ATOMIC_MAXBYTES_LOCK_FREE >= 8) */

#define __APPEND2(A,B) A##B
#define __APPEND(A,B) __APPEND2(A,B)
#if !_ATOMIC_FLAG_USES_LOCK

__INLINE
int _Atomic_flag_test_and_set(volatile _Atomic_flag_t *_Flag,
	memory_order _Order)
{
    return __APPEND(_Atomic_exchange_,_ATOMIC_FLAG_SIZE)(_Flag, 1, _Order);
}

__INLINE
void _Atomic_flag_clear(volatile _Atomic_flag_t *_Flag, memory_order _Order)
{
     __APPEND(_Atomic_store_,_ATOMIC_FLAG_SIZE)(_Flag, 0, _Order);
}
#endif /* _ATOMIC_FLAG_USES_LOCK */

#if !_ATOMIC_FENCE_USES_LOCK
__INLINE
void _Atomic_thread_fence(memory_order _Order)
{
    if (_Order != memory_order_relaxed) {
#if defined(_ATOMIC_FENCE_USES_EXCHANGE)
	static _Atomic_flag_t guard;
	__APPEND(_Atomic_exchange_,_ATOMIC_FLAG_SIZE)(&guard, 0, _Order);
#else
    __GHS_ATOMIC_FENCE(_POS_FENCE, _LDST_LOAD, _Order, 0);
#endif
    }
}

__INLINE
void _Atomic_signal_fence(memory_order _Order)
{    
    __schedule_barrier();
}

#endif /* _ATOMIC_FENCE_USES_LOCK */

#undef __APPEND2
#undef __APPEND

_STD_END
#undef __INLINE
#endif /* __OUT_OF_LINE_ATOMICS */

#ifdef __cplusplus
} /* extern "C" */
#endif /* __cplusplus */
#endif /* __ATOMIC_GHS_H */