ImageCuda.hpp

Go to the documentation of this file.

//
// This file is part of MorphoGraphX - http://www.MorphoGraphX.org
// Copyright (C) 2012-2016 Richard S. Smith and collaborators.
//
// If you use MorphoGraphX in your work, please cite:
//   http://dx.doi.org/10.7554/eLife.05864
//
// MorphoGraphX is free software, and is licensed under under the terms of the 
// GNU General (GPL) Public License version 2.0, http://www.gnu.org/licenses.
// 
#ifndef IMAGE_CUDA_HPP
#define IMAGE_CUDA_HPP
 
#include <Cuda.hpp>
 
namespace mgx
{
  // Find a 1D Gaussian mask (normalized to sum up to 1)
  inline bool getGaussianMask1D(const float step, const float sigma, HostVf& mask, int& radius)
  {
    // Find mask
    float sigmaSqr = sigma * sigma;
    float rad = .5 + 3.0 * sigma / step;
    radius = (rad <= 0 ? 0 : int(rad));
    if(radius <= 0)
      return false;
  
    float normalize = 0;
    mask.resize(2 * radius + 1);
    int j = 0;
    for(int u = -radius; u <= radius; ++u) {
      float x = u * step;
      double alpha = exp(-(x * x) / (2.0 * sigmaSqr));
      float val = alpha / (2.0 * M_PI * sigma);
      mask[j++] = val;
      normalize += val;
    }
    if(normalize <= 0)
      return false;
    for(size_t i = 0; i < mask.size(); ++i)
      mask[i] /= normalize;
 
    return true;
  }
 
  // Create a mask for a ellipsoid neighborhood
  inline bool getRoundMask3D(const Point3i &radius, HostVf &mask)
  {
    Point3i size = radius * 2 + Point3i(1,1,1);
    mask.resize(size.x() * size.y() * size.z());
  
    // Setup mask (we could antialias here)
    for(int z = 0; z < size.z(); ++z)
      for(int y = 0; y < size.y(); ++y)
        for(int x = 0; x < size.x(); ++x) {
          float xdist = float(x - radius.x())/float(radius.x());
          float ydist = float(y - radius.y())/float(radius.y());
          float zdist = float(z - radius.z())/float(radius.z());
          int offset = getOffset(x, y, z,  size);
          if(sqrt(xdist*xdist + ydist*ydist + zdist*zdist) <= 1.0)
            mask[offset] = 1;
          else
            mask[offset] = 0;
        }
    return true;
  } 
 
  // Normalize 1D Mask
  inline void normalizeMaskScale(HostVf& mask)
  {
    if(mask.empty())
      return;
  
    float normalize = 0;
    for(size_t i = 0; i < mask.size(); ++i)
      normalize += mask[i];
  
    if(normalize > 0)
      for(size_t i = 0; i < mask.size(); ++i)
        mask[i] /= normalize;
  }
  
//  // Go from image coordinates to world coordinates (centers of voxels)
//  __device__ __host__ inline 
//  Point3f imageToWorld(const Point3i& img, const Point3f& step, const Point3f& shift)
//  {
//    return multiply(Point3f(img) + Point3f(.5f, .5f, .5f), step) + shift;
//  }
//  
//  // Go from world coordinates to image coordinates
//  __device__ __host__ inline 
//  Point3i worldToImage(const Point3f& wrld, const Point3f& step, const Point3f& shift)
//  {
//    return Point3i((wrld - shift) / step - Point3f(.5f, .5f, .5f));
//  }
//  
//  // Compute offset in stack
//  __device__ __host__ inline size_t offset(uint x, uint y, uint z, uint xsz, uint ysz)
//  {
//    return ((size_t(z) * ysz + y) * xsz + x);
//  }
//  
//  __device__ __host__ inline 
//  size_t getOffset(const int x, const int y, const int z, const Point3i& size)
//  {
//    return ((size_t(z) * size.y() + y) * size.x() + x);
//  }
//  
//  __device__ __host__ inline size_t getOffset(const Point3i p, const Point3i& size)
//  {
//    return ((size_t(p.z()) * size.y() + p.y()) * size.x() + p.x());
//  }
  __device__ __host__ inline void getXY(uint idx, const Point3i& size, int& x, int& y)
  {
    x = idx % size.x();
    y = idx / size.x();
  }
  
  __device__ __host__ inline void getXZ(uint idx, const Point3i& size, int& x, int& z)
  {
    x = idx % size.x();
    z = idx / size.x();
  }
  
  __device__ __host__ inline void getYZ(uint idx, const Point3i& size, int& y, int& z)
  {
    y = idx % size.y();
    z = idx / size.y();
  }
  
  __device__ __host__ inline void getXYZ(uint idx, const Point3i& size, int& x, int& y, int& z)
  {
    x = idx % size.x();
    y = idx / size.x() % size.y();
    z = idx / size.x() / size.y();
  }
  
  // Compute stride
  __device__ __host__ inline Point3i getStride(const Point3i& size)
  {
    return (Point3i(1, size.x(), size.x() * size.y()));
  }
  
  // Clip point to clipping planes
  __device__ __host__ inline 
  bool clipPoint(const Point3f& wp, const Point3u& clipDo, const Point4f* pn)
  {
    bool clip = false;
    Point4f p4 = Point4f(wp.x(), wp.y(), wp.z(), 1.0);
    for(int i = 0; i < 3; i++)
      if(clipDo[i]) {
        // Test if inside clipping planes
        for(int k = 0; k < 2; k++)
          if(pn[i * 2 + k] * p4 < 0)
            clip = true;
      }
    return (clip);
  }
  
  // --- Cuda kernels ---
  
  // Average
  struct AverageKernel
  {
    int acc;
    int count;
  
    __device__ __host__ void init() { acc = count = 0; }
  
    __device__ __host__ void data(int val, int) { acc += val; count++; }
  
    __device__ __host__ ushort result() { return ushort(acc / count); }
  };
 
  // Median
  struct MedianKernel
  {
    static const int MaxMedianMask = 7*7*7;
    int array[MaxMedianMask];
    int count;
 
    __device__ __host__ void init() { count = 0; }
  
    __device__ __host__ void data(int val, int) { array[count++] = val; }
 
    // quickselect
    __device__ __host__ ushort select(ushort rank)
    {
      ushort low = 0, high = count - 1;
      int tmpVal;
#define SWAP(x,y) { tmpVal = (x); (x) = (y); (y) = tmpVal; }
      while(true)
      {
        if(high <= low) return array[rank]; // catches rank == pivot (below)
        if(high == low + 1)
        {
          if(array[low] > array[high]) SWAP(array[low],array[high]);
          return array[rank];
        }
 
        // Middle is halfway between low and high
        // Sort the three elements in low, middle, and high
        // Put the medium one out of the way in low and the lowest in low+1
        ushort middle = (low + high) / 2;
        if(array[middle] > array[high]) SWAP(array[middle],array[high]);
        if(array[low] > array[high]) SWAP(array[low],array[high]);
        if(array[middle] > array[low]) SWAP(array[low],array[middle]);
        SWAP(array[middle],array[low+1]);
 
        // Pivot around median element (currently in low)
        ushort ll = low+1, hh = high;
        while(true)
        {
          do ll++; while (array[low] > array[ll]);
          do hh--; while (array[hh] > array[low]);
          if(hh < ll) break;
          SWAP(array[ll],array[hh]);
        }
 
        // Swap median back to the correct position
        SWAP(array[low],array[hh]);
 
        // Shift low and high as needed
        if(rank <= hh) high = hh-1;
        if(rank >= hh) low = ll;
      }
#undef SWAP
    }
  
    __device__ __host__ ushort result() { return select(count / 2); }
  };
  
  // Dilate
  struct DilateKernel
  {
    int acc;
  
    __device__ __host__ void init() { acc = 0; }
  
    __device__ __host__ void data(int val, int)
    {
      if(val > acc)
        acc = val;
    }
  
    __device__ __host__ ushort result() { return ushort(acc); }
  };
 
  // Dilate with generic mask
  struct DilateMaskKernel
  {
    float acc;
    const float* mask;
  
    DilateMaskKernel(const float* _mask) : mask(_mask) {}
  
    __device__ __host__ void init() { acc = 0; }
  
    __device__ __host__ void data(int val, int pos) 
    { 
      if(mask[pos] > 0 and val > acc)
        acc = val; 
    }
  
    __device__ __host__ ushort result() { return ushort(acc); }
  };
 
  // Erode
  struct ErodeLabelsKernel
  {
    int acc;
  
    __device__ __host__ void init() { acc = -1; }
  
    __device__ __host__ void data(int val, int)
    {
      if(acc < 0)
        acc = val;
      else if(val != acc)
        acc = 0;
    }
  
    __device__ __host__ ushort result() { return ushort(acc); }
  };
  
  // Erode
  struct ErodeKernel
  {
    int acc;
  
    __device__ __host__ void init() { acc = 65536; }
  
    __device__ __host__ void data(int val, int)
    {
      if(val < acc)
        acc = val;
    }
  
    __device__ __host__ ushort result() { return ushort(acc); }
  };
 
  // Erode with generic mask
  struct ErodeMaskKernel
  {
    float acc;
    const float* mask;
  
    ErodeMaskKernel(const float* _mask) : mask(_mask) {}
  
    __device__ __host__ void init() { acc = 65536; }
  
    __device__ __host__ void data(int val, int pos) 
    { 
      if(mask[pos] > 0 and val < acc)
        acc = val; 
    }
  
    __device__ __host__ ushort result() { return ushort(acc); }
  };
 
  // Erode with mask (used for spherical neighborhood)
  template <typename T>
  struct MaskKernel
  {
    float acc;
    const float* mask;
  
    MaskKernel(const float* _mask) : mask(_mask) {}
  
    __device__ __host__ void init() { acc = 0; }
  
    __device__ __host__ void data(T val, int pos) { acc += val * mask[pos]; }
  
    __device__ __host__ T result()
    {
      // Trim result
      if(acc <= std::numeric_limits<T>::lowest())
        return std::numeric_limits<T>::lowest();
      else if(acc >= std::numeric_limits<T>::max())
        return std::numeric_limits<T>::max();
      else if(std::numeric_limits<T>::is_integer)
        return T(acc + 0.5); // Round integer types
      else
        return T(acc);
    }
  };
 
  // Separable operation data along X axis
  template <typename opT, typename T> struct sepIterX 
  {
    opT pixOP;
    int radius;
  
    Point3i size;
    Point3i stride;
    T* Src;
    T* Dst;
 
    sepIterX(opT _pixOP, int _radius, 
             thrust::device_vector<T> &_Src, thrust::device_vector<T> &_Dst)
      : pixOP(_pixOP), radius(_radius), Src(devP(_Src)), Dst(devP(_Dst)) {}
  
    void setSizes(const Point3i&, const Point3i& _size)
    {
      size = _size;
      stride = getStride(size);
    }
  
    __device__ __host__ void operator()(uint idx)
    {
      opT pxOP = pixOP;
      int x, y;
      getXY(idx, size, x, y);
  
      int xs = x - radius, xe = x + radius + 1;
      if(xs < 0)
        xs = 0;
      if(xe > size.x())
        xe = size.x();
      size_t zIdx = getOffset(x, y, 0, size);
      int maskinit = radius - (x - xs);     // Calculate index into mask
  
      for(int z = 0; z < size.z(); ++z, zIdx += stride.z()) {
        pxOP.init();
        size_t xIdx = getOffset(xs, y, z, size);
        int maskpos = maskinit;
        for(int xi = xs; xi < xe; ++xi, xIdx += stride.x())
          pxOP.data(Src[xIdx], maskpos++);
        Dst[zIdx] = pxOP.result();
      }
    }
  };
  
  // Separable operation along Y axis
  template <typename opT, typename T> struct sepIterY 
  {
    opT pixOP;
    int radius;
  
    Point3i size;
    Point3i stride;
    T* Src;
    T* Dst;
  
    sepIterY(opT _pixOP, int _radius, 
                   thrust::device_vector<T>& _Src, thrust::device_vector<T>& _Dst)
      : pixOP(_pixOP), radius(_radius), Src(devP(_Src)), Dst(devP(_Dst)) {}
  
    void setSizes(const Point3i&, const Point3i& _size)
    {
      size = _size;
      stride = getStride(size);
    }
  
    __device__ __host__ void operator()(uint idx)
    {
      opT pxOP = pixOP;
      int x, y;
      getXY(idx, size, x, y);
  
      int ys = y - radius, ye = y + radius + 1;
      if(ys < 0)
        ys = 0;
      if(ye > size.y())
        ye = size.y();
      size_t zIdx = getOffset(x, y, 0, size);
      int maskinit = radius - (y - ys);     // Calculate index into mask
  
      for(int z = 0; z < size.z(); ++z, zIdx += stride.z()) {
        pxOP.init();
        int maskpos = maskinit;
        size_t yIdx = getOffset(x, ys, z, size);
        for(int yi = ys; yi < ye; ++yi, yIdx += stride.y())
          pxOP.data(Src[yIdx], maskpos++);
        Dst[zIdx] = pxOP.result();
      }
    }
  };
  
  // Separable operation along Z axis with XY-slices
  template <typename opT, typename T> struct sepIterZ 
  {
    opT pixOP;
    int radius;
  
    Point3i size;
    Point3i stride;
    T* Src;
    T* Dst;
  
    sepIterZ(opT _pixOP, int _radius, 
                      thrust::device_vector<T>& _Src, thrust::device_vector<T>& _Dst)
      : pixOP(_pixOP), radius(_radius), Src(devP(_Src)), Dst(devP(_Dst)) {}
  
    void setSizes(const Point3i&, const Point3i& _size)
    {
      size = _size;
      stride = getStride(size);
    }
  
    __device__ __host__ void operator()(uint idx)
    {
      opT pxOP = pixOP;
      int x, y;
      getXY(idx, size, x, y);
  
      size_t zOff = getOffset(x, y, 0, size);
      for(int z = 0; z < size.z(); ++z, zOff += stride.z()) {
        int zs = z - radius, ze = z + radius + 1;
        if(zs < 0)
          zs = 0;
        if(ze > size.z())
          ze = size.z();
  
        pxOP.init();
        int maskpos = radius - (z - zs);       // Calculate index into mask
  
        size_t zIdx = getOffset(x, y, zs, size);
        for(int zi = zs; zi < ze; ++zi, zIdx += stride.z())
          pxOP.data(Src[zIdx], maskpos++);
        Dst[zOff] = pxOP.result();
      }
    }
  };
  
  // Flip an image
  template <typename T> void flipImage(DimEnum dim, Point3i imgSize, 
                                        Point3i& flipImgSize, const T& src, T& dst)
  {
    if(dim == XY) {
      flipImgSize = Point3i(imgSize.y(), imgSize.x(), imgSize.z());
      size_t idx = 0;
      for(int z = 0; z < imgSize.z(); ++z)
        for(int y = 0; y < imgSize.y(); ++y)
          for(int x = 0; x < imgSize.x(); ++x)
            dst[getOffset(y, x, z, flipImgSize)] = src[idx++];
    } else if(dim == YZ) {
      flipImgSize = Point3i(imgSize.x(), imgSize.z(), imgSize.y());
      size_t idx = 0;
      for(int z = 0; z < imgSize.z(); ++z)
        for(int y = 0; y < imgSize.y(); ++y)
          for(int x = 0; x < imgSize.x(); ++x)
            dst[getOffset(x, z, y, flipImgSize)] = src[idx++];
    } else if(dim == XZ) {
      flipImgSize = Point3i(imgSize.z(), imgSize.y(), imgSize.x());
      size_t idx = 0;
      for(int z = 0; z < imgSize.z(); ++z)
        for(int y = 0; y < imgSize.y(); ++y)
          for(int x = 0; x < imgSize.x(); ++x)
            dst[getOffset(z, y, x, flipImgSize)] = src[idx++];
    } else
      std::cout << "Error in flipImage: unknown dimension" << std::endl;
  }
  
  // Calculate threads
  inline uint getTotalThreads(DimEnum dim, const Point3i size)
  {
    switch(dim) {
    case X:
      return size.x();
    case Y:
      return size.y();
    case Z:
      return size.z();
    case XY:
      return size.x() * size.y();
    case XZ:
      return size.x() * size.z();
    case YZ:
      return size.y() * size.z();
    case XYZ:
      size_t threads = size_t(size.x()) * size.y() * size.z();
      if(threads > UINT_MAX)
        throw(std::string("Max threads greater than unsigned integer capacity, reduce holdmem"));
      return threads;
    }
    return 0;
  }
  
  // Check bounds
  inline int checkBounds(const Point3i& imgSize, const Point3i& base, const Point3i& size)
  {
    if(base.x() < 0 || base.y() < 0 || base.z() < 0)
      return errMsg("checkbounds:base < 0");
    if(size.x() <= 0 || size.y() <= 0 || size.z() <= 0)
      return errMsg("checkbounds:size =  0");
    if(base.x() + size.x() > imgSize.x() || base.y() + size.y() > imgSize.y() || 
                                                      base.z() + size.z() > imgSize.z())
      return errMsg("checkbounds:size > image size");
  
    return (0);
  }
  
  // Divide stack into sections to limit memory usage.
  inline uint getStep(uint pos, uint maxpos, size_t stride, size_t szof, size_t mem)
  {
    if(pos >= maxpos)
      return (0);
  
    // Check parms
    if(stride * szof == 0)
      throw(std::string("getStep: Bad stride (=0)"));
  
    // See how many slices can be done, do in chunks to limit memory usage.
    uint step = mem / (stride * szof);
    if(step == 0)
      throw(std::string("getStep: Can't process enough data, try increasing CudaHoldMem"));
  
    if(pos + step > maxpos)
      step = maxpos - pos;
  
    return (step);
  }
  
  // Class to process generic command using XY blocks with padding
  class processOP 
  {
  public:
    processOP(Point3i _imgSize, int _pad, size_t _sizeOf)
      : sizeOf(_sizeOf), imgSize(_imgSize), pad(_pad), baseZ(0)
    {
      mem = userMem();     // Set at start and don't change
      calcSize();
      passesReq = ceil(float(imgSize.z()) / sizeZ);
      if(passesReq > 1)
        std::cout << "Blocking data to GPU, " 
          << int(1.1 * sizeOf * imgSize.x() * imgSize.y() * imgSize.z() / (1024 * 1024)) 
          << " MB required, " << mem / MByte << " MB available (" 
          << sizeZ << " slices)" << std::endl;
    }
  
    template <typename T> void alloc(T& Dvec)
    {
      // Setup device storage
      size_t size = size_t(padSizeZ) * imgSize.x() * imgSize.y();
      if(Dvec.size() != size) {
        Dvec.resize(size);
        checkCudaError("processOp.alloc:Error allocating device vector.");
      }
    }
  
    template <typename HT, typename DT> void write(const HT& Hvec, DT& Dvec)
    {
      // Copy source data to device
      alloc(Dvec);
      thrust::copy(Hvec.begin() + size_t(padBaseZ) * imgSize.x() * imgSize.y(),
                   Hvec.begin() + size_t(padBaseZ + padSizeZ) * imgSize.x() * imgSize.y(), Dvec.begin());
      checkCudaError("processOp.write:Error copying data to device.");
    }
  
    // Launch threads to process a command that has the device vectors already allocated
    template <typename T> bool launch(T op, DimEnum dim)
    {
      // Set up operator pointers to data
      Point3i base(0, 0, padBaseZ);
      op.setSizes(base, Point3i(imgSize.x(), imgSize.y(), padSizeZ));
 
      uint maxThreads = 0;
      uint totThreads = getTotalThreads(dim, Point3i(imgSize.x(), imgSize.y(), padSizeZ));
      for(uint threadPos = 0; threadPos < totThreads; ) {
        // Calculate threads
        uint threads = getThreadCount(threadPos, totThreads);
        if(maxThreads < threads)
          maxThreads = threads;
        // Launch threads
        DCountIter first(threadPos), last(threadPos + threads);
        thrust::for_each(first, last, op);
        checkCudaError("processOp.launchDeviceOP:launch failure.");
        if(!progressAdvance())
          return errMsg("Operation canceled");
        threadPos += threads;
      }
      return false;
    }
  
    template <typename HT, typename DT> void read(const DT& Dvec, HT& Hvec)
    {
      // Copy results back to host, only works if x and y are not padded.
      thrust::copy(Dvec.begin() + size_t(baseZ - padBaseZ) * imgSize.x() * imgSize.y(),
                   Dvec.begin() + size_t(baseZ - padBaseZ + sizeZ) * imgSize.x() * imgSize.y(),
                   Hvec.begin() + size_t(baseZ) * imgSize.x() * imgSize.y());
      checkCudaError("processOp.read:Error copying data to host.");
    }
  
    int next()
    {
      baseZ += sizeZ;
      calcSize();
      return imgSize.z() - baseZ;
    }
  
    void rewind()
    {
      baseZ = 0;
      calcSize();
    }
  
    int passes() { return passesReq; }
  
  private:
    int calcSize()
    {
      // Finished, nothing to do
      if(baseZ >= imgSize.z())
        return 0;
  
      // Calculate padding at the base
      padBaseZ = baseZ;
      padBaseZ -= pad;
      if(padBaseZ < 0)
        padBaseZ = 0;
  
      // Calcaulate how much we can do.
      padSizeZ = getStep(padBaseZ, imgSize.z(), imgSize.x() * imgSize.y(), sizeOf, mem);
  
      // Adjust size for padding
      sizeZ = padSizeZ;
      if(padBaseZ + padSizeZ < imgSize.z())
        sizeZ -= pad;
      sizeZ -= baseZ - padBaseZ;
  
      // Not enough memory for one pass
      if(sizeZ <= 0)
        throw(std::string("processOP.calcSize: Can't process enough data, "
                                                        "try increasing cudaHoldMem"));
  
      return sizeZ;
    }
  
    size_t sizeOf;
    size_t mem;
    Point3i imgSize;
    int pad;
    int baseZ;
    int sizeZ;
    int padBaseZ;
    int padSizeZ;
    int passesReq;
  };
  
  // Process separable operators
  template <typename opT, typename T>
  int processSepOP(opT opx, opT opy, opT opz, const Point3i& imgSize, const Point3i& radius,
                   const thrust::host_vector<T>& srcData, thrust::host_vector<T>& dstData)
  {
    if(radius.x() < 0 && radius.y() < 0 && radius.z() < 0)
      throw(std::string("processSepOP: At least one radius must be >= 0"));
  
    if(radius.x() == 0 && radius.y() == 0 && radius.z() == 0) {
      dstData = srcData;
      return 0;
    }
    thrust::host_vector<T> Hsrc(srcData);
  
    // Loop over sections of XY slices
    processOP proc(imgSize, radius.z(), sizeof(T) * 2);
    thrust::device_vector<T> DVec1, DVec2;
    thrust::device_vector<T> *Dsrc = &DVec1, *Ddst = &DVec2;
    do {
      proc.write(Hsrc, *Dsrc);
      proc.alloc(DVec2);
  
      if(radius.x() > 0) {
        if(proc.launch(sepIterX<opT, T>(opx, radius.x(), *Dsrc, *Ddst), XY))
          return 1;
        std::swap(Dsrc, Ddst);
      }
      if(radius.y() > 0) {
        if(proc.launch(sepIterY<opT, T>(opy, radius.y(), *Dsrc, *Ddst), XY))
          return 1;
        std::swap(Dsrc, Ddst);
      }
      if(radius.z() > 0) {
        if(proc.launch(sepIterZ<opT, T>(opz, radius.z(), *Dsrc, *Ddst), XY))
          return 1;
        std::swap(Dsrc, Ddst);
      }
  
      proc.read(*Dsrc, dstData);
    } while(proc.next());
  
    return (0);
  }
 
  // Non separable operation, calculates a column (in z) of voxels
  template <typename opT, typename T>
  struct nonSepIter
  {
    opT pixOP;
    Point3i radius;
    Point3i imgSize;
    T *Src;
    T *Dst;
  
    nonSepIter(opT _pixOP, Point3i _radius, 
                  thrust::device_vector<T> &_Src, thrust::device_vector<T> &_Dst) 
      : pixOP(_pixOP), radius(_radius), Src(devP(_Src)), Dst(devP(_Dst)) {} 
  
    void setSizes(const Point3i _base, const Point3i &_imgSize)
    {
      imgSize = _imgSize;
    }
    
    __device__ __host__
    void operator()(uint idx)
    {
      opT pxOP = pixOP;
      int x, y;
      getXY(idx, imgSize, x, y);
  
      // Calculate x and y neighborhood range
      int xs = x - radius.x(), xe = x + radius.x() + 1;
      if(xs < 0) xs = 0;
      if(xe > imgSize.x()) xe = imgSize.x();
  
      int ys = y - radius.y(), ye = y + radius.y() + 1;
      if(ys < 0) ys = 0;
      if(ye > imgSize.y()) ye = imgSize.y();
  
      // Index of first voxel to calculate
      Point3i maskSize = radius * 2 + Point3i(1,1,1);
  
      // Do a column of voxels in z 
      for(int z = 0; z < imgSize.z(); ++z) {
        pxOP.init();
  
        // Calculate x and y neighborhood range
        int zs = z - radius.z(), ze = z + radius.z() + 1;
        if(zs < 0) zs = 0;
        if(ze > imgSize.z()) ze = imgSize.z();
  
        for(int zi = zs; zi < ze; ++zi)
          for(int yi = ys; yi < ye; ++yi)
            for(int xi = xs; xi < xe; ++xi)
              pxOP.data(Src[getOffset(xi,yi,zi,imgSize)], getOffset(xi-xs,yi-ys,zi-zs,maskSize));
  
        Dst[getOffset(x,y,z,imgSize)] = pxOP.result();
      }
    }
  };
 
  // Process separable operators
  template <typename opT, typename T>
  int processNonSepOP(opT op, const Point3i &imgSize, const Point3i &radius,
                   const thrust::host_vector<T> &srcData, thrust::host_vector<T> &dstData)
  {
    if(radius.x() < 0 && radius.y() < 0 && radius.z() < 0)
      throw(std::string("processSepOP: At least one radius must be >= 0"));
  
    if(radius.x() == 0 && radius.y() == 0 && radius.z() == 0) {
      dstData = srcData;
      return 0;
    }
    thrust::host_vector<T> Hsrc(srcData);
  
    // Loop over sections of XY slices
    processOP proc(imgSize, radius.z(), sizeof(T) * 2);
    thrust::device_vector<T> DVec1, DVec2;
    thrust::device_vector<T>* Dsrc = &DVec1, *Ddst = &DVec2;
    do {
      proc.write(Hsrc, *Dsrc);
      proc.alloc(DVec2);
  
      if(proc.launch(nonSepIter<opT, T>(op, radius, *Dsrc, *Ddst), XY))
        return 1;
      std::swap(Dsrc, Ddst);
  
      proc.read(*Dsrc, dstData);
    } while(proc.next());
  
    return (0);
  }
}
#endif