http://folding.stanford.edu/English/FAQ-flops

They give lots of details and are very clear, in bref, a good reference.

set(CMAKE_CXX_COMPILER g++-4.4)
set(CMAKE_CC_COMPILER gcc-4.4)

#The name of the project
PROJECT(DIRECT_CUDA)

#The FindCUDA script is distributed since version 2.8
CMAKE_MINIMUM_REQUIRED(VERSION 2.8)

FIND_PACKAGE(CUDA)

INCLUDE(FindCUDA)

CUDA_ADD_EXECUTABLE(direct.exe direct.cu main.cpp)

I use Gcc4.4 because my cuda library does not work with Gcc4.6, you in the future of course it may not be needed anymore.

First declare  the block of data in your .cu file:

// in the .cu file

__constant__ int ConstArray[512];

// use it as any normal array in your cuda functions

Then in your .c/.cpp file, you do not need to allocate using cudaMalloc since the block as already been declared, you just need to copy from host to device using cudaMemcpyToSymbol.

int main(){
	int array[512];

	// work with array

	cudaMemcpyToSymbol("ConstArray", array, sizeof(int) * 512 );

	// call your gpu func

	return 0;
}

 OctoBooth is a website that creates facial transformations

Change race, age or sex, and share photos with your friends

OctoBooth is free, and no registration is required

Enjoy.

Here I developed a quick sort based on the great qt feature QtConcurrent.
It is mostly similar to the openmp tasks versions.

Advices

Change the fallowing lines in the code to perform different tests

QThreadPool::globalInstance()->setMaxThreadCount(1);
const long Size = 10000000;

The code

#include <QtCore/QCoreApplication>
#include <QTime>
#include <QtConcurrentRun>

#include <iostream>

////////////////////////////////////////////////////////////
// Miscialenous functions
////////////////////////////////////////////////////////////

/** Swap to value */
template <class NumType>
inline void Swap(NumType& value, NumType& other){
    NumType temp = value;
    value = other;
    other = temp;
}

////////////////////////////////////////////////////////////
// Quick sort
////////////////////////////////////////////////////////////

/* use in the sequential qs */
template <class SortType>
long QsPartition(SortType outputArray[], long left, long right){
    const long part = right;
    Swap(outputArray[part],outputArray[left + (right - left ) / 2]);
    const SortType partValue = outputArray[part];
    --right;

    while(true){
        while(outputArray[left] < partValue){
            ++left;
        }
        while(right >= left && partValue <= outputArray[right]){
            --right;
        }
        if(right < left) break;

        Swap(outputArray[left],outputArray[right]);
        ++left;
        --right;
    }

    Swap(outputArray[part],outputArray[left]);

    return left;
}

/* a sequential qs */
template <class SortType>
void QsSequential(SortType array[], const long left, const long right){
    if(left < right){
        const long part = QsPartition(array, left, right);
        QsSequential(array,part + 1,right);
        QsSequential(array,left,part - 1);
    }
}

/** A task dispatcher */
template <class SortType>
void QuickSortTask(SortType array[], const long left, const long right, const int deep){
    if(left < right){
        if( deep ){
            const long part = QsPartition(array, left, right);

            QtConcurrent::run(QuickSortTask<SortType>, array, part + 1, right, deep - 1);
            QtConcurrent::run(QuickSortTask<SortType>, array, left, part - 1, deep - 1);
        }
        else {
            const long part = QsPartition(array, left, right);
            QsSequential(array,part + 1,right);
            QsSequential(array,left,part - 1);
        }
    }
}

////////////////////////////////////////////////////////////
// Main
////////////////////////////////////////////////////////////

template <class SortedType>
bool isSorted(SortedType array[], const long size){
    for(int idx = 1; idx < size ; ++idx){
        if(array[idx-1] > array[idx]){
            return false;
        }
    }
    return true;
}

template <class SortedType>
void print(SortedType array[], const int size){
    for(int idx = 0 ;idx < size; ++idx){
        std::cout << array[idx] << "\t";
    }
    std::cout << "\n";
}

int main(int argc, char** argv){
    QCoreApplication app(argc, argv);
    // Change to test efficiency
    // QThreadPool::globalInstance()->setMaxThreadCount(1);

    const long Size = 10000000;//600000000;
    long* const array = new long[Size];

    // Create array
    srand(0);
    for(long idx = 0 ; idx < Size ; ++idx){
        array[idx] = int(Size*(float(rand())/RAND_MAX));
    }

    printf("Sorting %ld elements\n", Size);
    // Start sorting
    QTime timer;
    timer.start();
    QtConcurrent::run(QuickSortTask<long>, array, 0, Size - 1, 6);
    QThreadPool::globalInstance()->waitForDone();
    printf("Elapsed time %f s\n", timer.elapsed()/1000.0);

    // Test result
    if(isSorted(array,Size)){
        printf("Is sorted\n");
    }
    else{
        printf("Error array is not sorted!\n");
        if( Size <= 20) print(array,Size);
        return -1;
    }

    // remove array and quit
    delete [] array;
    return 0;
}

Licence : lgpl.

It has been done quickly, it is an unpretentious document.
Of course an entire book about efficient C++ is better.

Get the doc here (pdf)

What

I developed a basic application that goes from one image to another in a given time step (4s but it is customizable).
Lets say we have 3 pixels, source (from source image) dest (from destination image) and frame (from the computed image).

• At time 0: frame = source
• At time STEP: frame = dest
• At time STEP/2: frame is between source and dest.

More generally,
frame(r,g,b) = (dest(r,g,b) – source(r,g,b)) * percent + source(r,g,b)

Examples:

The application

So the application lets you choose the images, choose the type of computation (thread,sequential,gpu) and start stop.

The efficency

Images size : 1024 x 768
Processors : i7 (4 x CPU) 2.8 GHz
Ubuntu : 11.04
Transfer Time : 4s
Sequential FPS : 6.3 f/s
Thread FPS : 25.s f/s
GPU FPS : 145.2 f/s

Processing Code

Sequential

void ImageWorker::runSequential(){
    stopFlag = false;

    int totalTime        = 0;
    int nbFrameProcessed = 0;

    // emit source
    emit result(sourceImage, 0);

    QTime timer;
    timer.start();
    // compute until TimeToDest
    while(!stopFlag && timer.elapsed() < TimeToDest){
        totalTime        += sequentialStep( (timer.elapsed()/float(TimeToDest)) );
        nbFrameProcessed += 1;
        const float fps   = nbFrameProcessed / (totalTime/1000.0);

        emit result(frame, fps);
    }

    // emit dest
    const float fps   = nbFrameProcessed / (totalTime/1000.0);
    emit result(destImage, fps);
}

int ImageWorker::sequentialStep(const float percent){
    QTime timer;
    timer.start();

    for( int idxX = 0; idxX < sourceImage.width() ; ++idxX){
        for( int idxY = 0; idxY < sourceImage.height() ; ++idxY){
            // ARGB quadruplet on the format #AARRGGBB convert into signed
            int sourceValue = static_cast<int>(sourceImage.pixel(idxX, idxY));
            int destValue = static_cast<int>(destImage.pixel(idxX, idxY));

            const int red   = ((((destValue >> 16) & 0xFF) - ((sourceValue >> 16) & 0xFF)) * percent) + ((sourceValue >> 16) & 0xFF);
            const int green = ((((destValue >>   & 0xFF) - ((sourceValue >>   & 0xFF)) * percent) + ((sourceValue >>   & 0xFF);
            const int blue  = ((((destValue >>  0) & 0xFF) - ((sourceValue >>  0) & 0xFF)) * percent) + ((sourceValue >>  0) & 0xFF);

            // set value into intermediate image
            frame.setPixel(idxX, idxY, qRgb(red, green, blue));
        }
    }

    return timer.elapsed();
}

QConcurent

void ImageWorker::runConcurrent(){
    stopFlag = false;

    QVector< QPair<int, int> > chunkSizes(QThread::idealThreadCount());
    const double aChunk = ceil(double(sourceImage.height())/QThread::idealThreadCount());
    chunkSizes[0] = QPair<int, int>(0, aChunk);
    for( int idxThread = 1 ; idxThread < QThread::idealThreadCount() ; ++idxThread){
        chunkSizes[idxThread].first  = chunkSizes[idxThread-1].second;
        chunkSizes[idxThread].second = (idxThread + 1) * aChunk;
    }
    chunkSizes[QThread::idealThreadCount() - 1].second = sourceImage.height();

    int totalTime        = 0;
    int nbFrameProcessed = 0;

    emit result(sourceImage, 0);

    QTime timer;
    timer.start();
    while(!stopFlag && timer.elapsed() < TimeToDest){
        totalTime        += concurrentStep( (timer.elapsed()/float(TimeToDest)), chunkSizes );
        nbFrameProcessed += 1;
        const float fps   = nbFrameProcessed / (totalTime/1000.0);

        emit result(frame, fps);
    }

    const float fps   = nbFrameProcessed / (totalTime/1000.0);
    emit result(destImage, fps);
}

void convertImages(const QImage& sourceImage, const QImage& destImage, QImage* const frame,
                   const float percent, const QPair<int, int>& realHeight){
    const int width = sourceImage.width();
    QRgb * const threadBuffer = new QRgb[width];

    static QMutex locker;

    for( int idxY = realHeight.first; idxY < realHeight.second ; ++idxY){
        for( int idxX = 0; idxX < width ; ++idxX){
            // ARGB quadruplet on the format #AARRGGBB
            int sourceValue = static_cast<int>(sourceImage.pixel(idxX, idxY));
            int destValue = static_cast<int>(destImage.pixel(idxX, idxY));

            const int red   = ((((destValue >> 16) & 0xFF) - ((sourceValue >> 16) & 0xFF)) * percent) + ((sourceValue >> 16) & 0xFF);
            const int green = ((((destValue >>   & 0xFF) - ((sourceValue >>   & 0xFF)) * percent) + ((sourceValue >>   & 0xFF);
            const int blue  = ((((destValue >>  0) & 0xFF) - ((sourceValue >>  0) & 0xFF)) * percent) + ((sourceValue >>  0) & 0xFF);

            threadBuffer[idxX] = qRgb(red, green, blue);
        }
        locker.lock();
        for( int idxX = 0; idxX < width ; ++idxX){
            frame->setPixel(idxX, idxY, threadBuffer[idxX]);
        }
        locker.unlock();
    }

    delete[] threadBuffer;
}

int ImageWorker::concurrentStep(const float percent, const QVector< QPair<int, int> >& chunkSizes){
    QTime timer;
    timer.start();

    for( int idxThread = 0 ; idxThread < QThread::idealThreadCount() ; ++idxThread){
        QtConcurrent::run(convertImages, sourceImage, destImage, &frame, percent, chunkSizes[idxThread]);
    }

    // QThreadPool::globalInstance()->activeThreadCount()
    QThreadPool::globalInstance()->waitForDone();

    return timer.elapsed();
}

QtOpencl

#include <qclcontext.h>
#include <qclprogram.h>
#include <qclkernel.h>
#include <qclimage.h>

void ImageWorker::runOpencl(){
    stopFlag = false;

    QCLContext context;

    if (!context.create())
        qFatal("Could not create OpenCL context");

    if (!context.create(QCLDevice::GPU))
        qFatal("Could not create OpenCL context");

    QCLProgram program = context.buildProgramFromSourceFile(QLatin1String(":/transferimage.cl"));

    QCLImage2D sourceImageBuffer = context.createImage2DCopy(sourceImage, QCLMemoryObject::ReadOnly);
    QCLImage2D destImageBuffer = context.createImage2DCopy(destImage, QCLMemoryObject::ReadOnly);
    QCLImage2D frameBuffer = context.createImage2DDevice(frame.format(), frame.size(), QCLMemoryObject::WriteOnly);

    QCLKernel compute = program.createKernel("transfer");
    compute.setGlobalWorkSize(sourceImage.size());
    compute.setLocalWorkSize(8, 8);

    int totalTime        = 0;
    int nbFrameProcessed = 0;

    emit result(sourceImage, 0);

    QTime timerFrame;
    QTime timer;
    timer.start();
    while(!stopFlag && timer.elapsed() < TimeToDest){
        timerFrame.start();
        compute(sourceImageBuffer, destImageBuffer, frameBuffer, (timer.elapsed()/float(TimeToDest)) );
        frameBuffer.read(&frame);

        totalTime        += timerFrame.elapsed();
        nbFrameProcessed += 1;
        const float fps   = nbFrameProcessed / (totalTime/1000.0);

        emit result(frame, fps);
    }

    const float fps   = nbFrameProcessed / (totalTime/1000.0);
    emit result(destImage, fps);
}

const sampler_t samp = CLK_ADDRESS_CLAMP_TO_EDGE |
                       CLK_FILTER_LINEAR;
__kernel void transfer(__read_only image2d_t sourceImage,
                       __read_only image2d_t destImage,
                       __write_only image2d_t frameImage,
                       float percent)
{
    int2 pos = (int2)(get_global_id(0), get_global_id(1));
    float4 sourceColor = read_imagef(sourceImage, samp, pos);
    float4 destColor = read_imagef(destImage, samp, pos);

    float4 frameColor;
    frameColor.x = ((destColor.x - sourceColor.x) * percent) + sourceColor.x;
    frameColor.y = ((destColor.y - sourceColor.y) * percent) + sourceColor.y;
    frameColor.z = ((destColor.z - sourceColor.z) * percent) + sourceColor.z;
    frameColor.w = sourceColor.w;

    write_imagef(frameImage, pos, clamp(frameColor, 0.0f, 1.0f));
}

Download the code

The code is here.

References

http://labs.qt.nokia.com/2010/04/07/using-opencl-with-qt/

http://doc.qt.nokia.com/opencl-snapshot/concurrent.html

Download what you need from NVidia website:
http://developer.nvidia.com/cuda-downloads

Take:
“CUDA Toolkit for Ubuntu Linux 10.10″ (gpucomputingsdk_4.0.17_linux)
“GPU Computing SDK – complete package including all code samples” (cudatoolkit_4.0.17_linux_32_ubuntu10.10.run)
And if you want (not recommended):
“Developer Drivers for Linux” (devdriver_4.0_linux_32_270.41.19.run)

Install NVidia Driver

I recommend not to install Official NVidia Driver!!
(I did the first time and had a lot of problems!)
Use the safest install:

sudo apt-get install linux-headers-generic
sudo apt-get install nvidia-current
sudo nvidia-xconfig

If you need to remove bad drivers:

sudo apt-get remove --purge nvidia-current

Anyway, if you prefer you can install official nvidia package:

sudo /etc/init.d/gdm stop
sudo sh devdriver_4.0_linux_32_270.41.19.run
reboot

Install SDK

SDK has to be downloaded from the NVidia offical website.

Run the downloaded package:

sudo sh cudatoolkit_4.0.17_linux_32_ubuntu10.10.run

Then change your path and log dirs:

gedit ~/.bashrc

And paste:

# opencl
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64:/usr/local/cuda/lib
export PATH=$PATH:/usr/local/cuda/lib:/usr/local/cuda/bin

(note, in case of 32bits you do not need /usr/local/cuda/lib64)

Restart your terminal or tape:

source ~/.bashrc

Install Examples

Examples have to be downloaded from the NVidia offical website.

sudo sh gpucomputingsdk_4.0.17_linux

I need to install some dependencies:

sudo apt-get install libxmu-dev libxmu6
sudo apt-get install freeglut3-dev

Then compile OpenCl examples (remplace user name with yours):

cd ~
sudo chown -R “user name” NVIDIA_GPU_Computing_SDK
sudo chmod -R 777 NVIDIA_GPU_Computing_SDK
cd NVIDIA_GPU_Computing_SDK
cd OpenCL
make

Run examples in (for example run oclNbody)

cd ~/NVIDIA_GPU_Computing_SDK/OpenCL/bin/linux/release
./oclNbody

References

http://vgerscorner.wordpress.com/2010/10/24/opencl-ubuntu-install-guide/

http://forums.nvidia.com/index.php?showtopic=87692h2

Based on the specification:
http://openmp.org/wp/openmp-specifications/
You can notice that the openmp vers. 3.0 has been released in 2008.05
From this information you can use task or not depending on the version of your openmp.

#include <cstdio>
#include <omp.h>

int main(){
#if _OPENMP >= 200805
    // I use tasks
    printf("_OPENMP >= 200805 (vers = %d)\n",_OPENMP);
#else
    // Tasks do not exist...
    printf("_OPENMP < 200805 (vers = %d)\n",_OPENMP);
#endif

    return 0;
}

Be aware that this version need a number of processes power of 2.

Reference

Library Support for Parallel Sorting in Scientific Computations
http://web.mst.edu/~ercal/387/P3/pr-proj-3.pdf
http://en.wikipedia.org/wiki/Bitonic_sort
http://www.iti.fh-flensburg.de/lang/algorithmen/sortieren/bitonic/oddn.htm
http://en.wikipedia.org/wiki/Bisection_method

The Code

#include <cstdio>
#include <cstdlib>
#include <cmath>

#include <mpi.h>

////////////////////////////////////////////////////////////////
// Sequential sort first!
////////////////////////////////////////////////////////////////

template <class SortType>
inline void swap(SortType& v1, SortType& v2){
    const SortType tmp = v1;
    v1 = v2;
    v2 = tmp;
}

template <class SortType, class CompareType, class FSize>
int partition(SortType* const array, FSize left, FSize right){
    SortType part = right;
    swap(array[part],array[(right+left) / 2]);
    --right;

    while(true){
        while( CompareType(array[left]) < CompareType(array[part])){
            ++left;
        }
        while(right >= left && CompareType(array[part]) <= CompareType(array[right])){
            --right;
        }
        if(right < left) break;

        swap(array[left],array[right]);
        ++left;
        --right;
    }

    swap(array[part],array[left]);

    return left;
}

template <class SortType, class CompareType, class FSize>
void qs(SortType* const array, const FSize left, const FSize right){
    if(left < right){
        const FSize part = partition<SortType,CompareType>(array, left, right);
        qs<SortType,CompareType>(array,part + 1,right);
        qs<SortType,CompareType>(array,left,part - 1);
    }
}

template <class SortType, class CompareType, class FSize>
void quick(SortType* const array, const FSize size){
    qs<SortType,CompareType>(array,0,size-1);
}

////////////////////////////////////////////////////////////////
// Bitonic parallel sort !
////////////////////////////////////////////////////////////////

// Mpi flag
static const int FlagMin = 5;
static const int FlagMax = 6;
static const int FlagMinMess = 4;
static const int FlagMaxMess = 3;

// This function exchange data with the other rank,
// its send the max value and receive min value
template <class SortType, class CompareType, class FSize>
void sendMaxAndGetMin(SortType array[], const FSize size, const int otherRank){
    FSize left  = -1;
    FSize right = size - 1;
    FSize pivot = left + (right - left + 1)/2;
    CompareType otherValue = -1;
    CompareType tempCompareValue = CompareType(array[pivot]);
    MPI_Sendrecv(&tempCompareValue,sizeof(CompareType),MPI_BYTE,otherRank,FlagMin,&otherValue,sizeof(CompareType),MPI_BYTE,otherRank,FlagMax,MPI_COMM_WORLD,MPI_STATUS_IGNORE);

    while( pivot != left && pivot != right  && array[pivot] != otherValue) {

        if( array[pivot] < otherValue ){
            left = pivot;
        }
        else {
            right = pivot;
        }
        pivot = left + (right - left + 1)/2;
        tempCompareValue = CompareType(array[pivot]);

        MPI_Sendrecv(&tempCompareValue,sizeof(CompareType),MPI_BYTE,otherRank,FlagMin,&otherValue,sizeof(CompareType),MPI_BYTE,otherRank,FlagMax,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
    }

    if( otherValue <= array[pivot] ){
        MPI_Sendrecv_replace(&array[pivot], (size - pivot) * sizeof(SortType) , MPI_BYTE,
                               otherRank, FlagMinMess, otherRank, FlagMaxMess,
                               MPI_COMM_WORLD, MPI_STATUS_IGNORE);

    }
    else if( array[pivot] < otherValue){
        if(pivot != size - 1){
            MPI_Sendrecv_replace(&array[pivot + 1], (size - pivot - 1) * sizeof(SortType) , MPI_BYTE,
                                   otherRank, FlagMinMess, otherRank, FlagMaxMess,
                                   MPI_COMM_WORLD, MPI_STATUS_IGNORE);
        }
    }

}

// This function exchange data with the other rank,
// its send the min value and receive max value
template <class SortType, class CompareType, class FSize>
void sendMinAndGetMax(SortType array[], const FSize size, const int otherRank){
    FSize left  = 0;
    FSize right = size ;
    FSize pivot = left + (right - left)/2;
    CompareType otherValue = -1;
    CompareType tempCompareValue = CompareType(array[pivot]);
    MPI_Sendrecv(&tempCompareValue,sizeof(CompareType),MPI_BYTE,otherRank,FlagMax,&otherValue,sizeof(CompareType),MPI_BYTE,otherRank,FlagMin,MPI_COMM_WORLD,MPI_STATUS_IGNORE);

    while(  pivot != left  && array[pivot] != otherValue) {

        if( array[pivot] < otherValue ){
            left = pivot;
        }
        else {
            right = pivot;
        }
        pivot = left + (right - left)/2;
        tempCompareValue = CompareType(array[pivot]);
        MPI_Sendrecv(&tempCompareValue,sizeof(CompareType),MPI_BYTE,otherRank,FlagMax,&otherValue,sizeof(CompareType),MPI_BYTE,otherRank,FlagMin,MPI_COMM_WORLD,MPI_STATUS_IGNORE);
    }

    if( array[pivot] <= otherValue ){
        MPI_Sendrecv_replace(&array[0], (pivot + 1) * sizeof(SortType) , MPI_BYTE,
                               otherRank, FlagMaxMess, otherRank, FlagMinMess,
                               MPI_COMM_WORLD, MPI_STATUS_IGNORE);
    }
    else if( otherValue < array[pivot]){
        if(pivot != 0){
            MPI_Sendrecv_replace(&array[0], (pivot) * sizeof(SortType) , MPI_BYTE,
                                   otherRank, FlagMaxMess, otherRank, FlagMinMess,
                                   MPI_COMM_WORLD, MPI_STATUS_IGNORE);
        }
    }
}

/*
From :

http://web.mst.edu/~ercal/387/P3/pr-proj-3.pdf

Parallel Bitonic Sort Algorithm for processor Pk (for k := 0 . . . P − 1)
d:= log P
// cube dimension
sort(local − datak ) // sequential sort
// Bitonic Sort follows
for i:=1 to d do
    window-id = Most Significant (d-i) bits of Pk
    for j:=(i-1) down to 0 do
        if((window-id is even AND j th bit of Pk = 0)
        OR (window-id is odd AND j th bit of Pk = 1))
            then call CompareLow(j)
        else
            call CompareHigh(j)
        endif
    endfor
endfor
  */
template <class SortType, class CompareType, class FSize>
void bitonic(SortType array[], const FSize size, const int np, const int rank){
    quick<SortType,CompareType>(array, size);

    const int logNp = log2(np);
    for(int bitIdx = 1 ; bitIdx <= logNp ; ++bitIdx){
        // window-id = Most Significant (d-i) bits of Pk
        const int diBit =  (rank >> bitIdx) & 0x1;

        for(int otherBit = bitIdx - 1 ; otherBit >= 0 ; --otherBit){
            // if((window-id is even AND j th bit of Pk = 0)
            // OR (window-id is odd AND j th bit of Pk = 1))

            const int myOtherBit = (rank >> otherBit) & 0x1;
            const int otherRank = rank ^ (1 << otherBit);

            if( diBit != myOtherBit ){
                sendMinAndGetMax<SortType,CompareType>(array, size, otherRank);
            }
            else{
                sendMaxAndGetMin<SortType,CompareType>(array, size, otherRank);
            }
            // A merge sort is possible since the array is composed
            // by two part already sorted, but we want to do this in space
            quick<SortType,CompareType>(array, size);
        }
    }
}

////////////////////////////////////////////////////////////////
// Utils
////////////////////////////////////////////////////////////////

template <class SortType, class FSize>
bool isSorted(const SortType array[], const FSize size){
    for(int idx = 1 ; idx < size ; ++idx){
        if( array[idx-1] > array[idx]){
            return false;
        }
    }
    return true;
}

void print(const int array[], const int size, const int rank){
    for(int idx = 0 ; idx < size ; ++idx){
        printf("array[%d][%d] = %d\n", rank, idx, array[idx]);
    }
}

int main(int argc, char ** argv){
    MPI_Init(&argc, &argv);

    int rank = 0;
    int nprocs = 0;

    MPI_Comm_size(MPI_COMM_WORLD,&nprocs);
    MPI_Comm_rank(MPI_COMM_WORLD,&rank);

    const int Size = 500;
    long long array[Size];
    srand(Size);

    for(int idx = 0 ; idx < Size ; ++idx){
        //array[idx] = nprocs - rank;
        //array[idx] = rank;
        array[idx] = Size * (rand()/float(RAND_MAX));
    }

    bitonic<long long,int>(array, Size, nprocs, rank);
    //print(array, Size, rank);

    int sorted = isSorted(array,Size);
    bool localySorted = false;
    MPI_Reduce( &sorted, &localySorted, 1, MPI_INT, MPI_LAND , 0, MPI_COMM_WORLD );

    int*const allExtrem = new int[nprocs * 2];
    int extrem[2];
    extrem[0] = array[0];
    extrem[1] = array[Size-1];
    MPI_Gather(extrem, 2, MPI_INT, allExtrem, 2, MPI_INT, 0, MPI_COMM_WORLD);

    printf(sorted?"Is sorted\n":"NO is not sorted\n");

    if( rank == 0){
        printf(localySorted?"All sorted\n":"NO all not sorted\n");

        int extremOk = true;
        for(int idxProc = 1 ; idxProc < nprocs && extremOk; ++idxProc){
            if( allExtrem[2 * (idxProc - 1) + 1] > allExtrem[2 * idxProc]){
                extremOk = false;
            }
        }

        printf(extremOk?"Extrem ok\n":"NO extrem error\n");
    }

    delete[] allExtrem;

    MPI_Finalize();

    return 0;
}