2015-08-28 15:33:40 +00:00
|
|
|
/* Plugin for NVPTX execution.
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
Copyright (C) 2013-2017 Free Software Foundation, Inc.
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
Contributed by Mentor Embedded.
|
|
|
|
|
|
|
|
This file is part of the GNU Offloading and Multi Processing Library
|
|
|
|
(libgomp).
|
|
|
|
|
|
|
|
Libgomp is free software; you can redistribute it and/or modify it
|
|
|
|
under the terms of the GNU General Public License as published by
|
|
|
|
the Free Software Foundation; either version 3, or (at your option)
|
|
|
|
any later version.
|
|
|
|
|
|
|
|
Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
|
|
|
|
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
|
|
|
FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
|
|
more details.
|
|
|
|
|
|
|
|
Under Section 7 of GPL version 3, you are granted additional
|
|
|
|
permissions described in the GCC Runtime Library Exception, version
|
|
|
|
3.1, as published by the Free Software Foundation.
|
|
|
|
|
|
|
|
You should have received a copy of the GNU General Public License and
|
|
|
|
a copy of the GCC Runtime Library Exception along with this program;
|
|
|
|
see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
|
|
|
|
<http://www.gnu.org/licenses/>. */
|
|
|
|
|
|
|
|
/* Nvidia PTX-specific parts of OpenACC support. The cuda driver
|
|
|
|
library appears to hold some implicit state, but the documentation
|
|
|
|
is not clear as to what that state might be. Or how one might
|
|
|
|
propagate it from one thread to another. */
|
|
|
|
|
|
|
|
#include "openacc.h"
|
|
|
|
#include "config.h"
|
|
|
|
#include "libgomp-plugin.h"
|
|
|
|
#include "oacc-plugin.h"
|
2017-04-10 11:32:00 +00:00
|
|
|
#include "gomp-constants.h"
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
#include <pthread.h>
|
|
|
|
#include <cuda.h>
|
|
|
|
#include <stdbool.h>
|
|
|
|
#include <stdint.h>
|
2017-10-07 00:16:47 +00:00
|
|
|
#include <limits.h>
|
2015-08-28 15:33:40 +00:00
|
|
|
#include <string.h>
|
|
|
|
#include <stdio.h>
|
|
|
|
#include <unistd.h>
|
|
|
|
#include <assert.h>
|
2017-10-07 00:16:47 +00:00
|
|
|
#include <errno.h>
|
|
|
|
|
|
|
|
#if PLUGIN_NVPTX_DYNAMIC
|
|
|
|
# include <dlfcn.h>
|
|
|
|
|
|
|
|
# define CUDA_CALLS \
|
|
|
|
CUDA_ONE_CALL (cuCtxCreate) \
|
|
|
|
CUDA_ONE_CALL (cuCtxDestroy) \
|
|
|
|
CUDA_ONE_CALL (cuCtxGetCurrent) \
|
|
|
|
CUDA_ONE_CALL (cuCtxGetDevice) \
|
|
|
|
CUDA_ONE_CALL (cuCtxPopCurrent) \
|
|
|
|
CUDA_ONE_CALL (cuCtxPushCurrent) \
|
|
|
|
CUDA_ONE_CALL (cuCtxSynchronize) \
|
|
|
|
CUDA_ONE_CALL (cuDeviceGet) \
|
|
|
|
CUDA_ONE_CALL (cuDeviceGetAttribute) \
|
|
|
|
CUDA_ONE_CALL (cuDeviceGetCount) \
|
|
|
|
CUDA_ONE_CALL (cuEventCreate) \
|
|
|
|
CUDA_ONE_CALL (cuEventDestroy) \
|
|
|
|
CUDA_ONE_CALL (cuEventElapsedTime) \
|
|
|
|
CUDA_ONE_CALL (cuEventQuery) \
|
|
|
|
CUDA_ONE_CALL (cuEventRecord) \
|
|
|
|
CUDA_ONE_CALL (cuEventSynchronize) \
|
|
|
|
CUDA_ONE_CALL (cuFuncGetAttribute) \
|
|
|
|
CUDA_ONE_CALL (cuGetErrorString) \
|
|
|
|
CUDA_ONE_CALL (cuInit) \
|
|
|
|
CUDA_ONE_CALL (cuLaunchKernel) \
|
|
|
|
CUDA_ONE_CALL (cuLinkAddData) \
|
|
|
|
CUDA_ONE_CALL (cuLinkComplete) \
|
|
|
|
CUDA_ONE_CALL (cuLinkCreate) \
|
|
|
|
CUDA_ONE_CALL (cuLinkDestroy) \
|
|
|
|
CUDA_ONE_CALL (cuMemAlloc) \
|
|
|
|
CUDA_ONE_CALL (cuMemAllocHost) \
|
|
|
|
CUDA_ONE_CALL (cuMemcpy) \
|
|
|
|
CUDA_ONE_CALL (cuMemcpyDtoDAsync) \
|
|
|
|
CUDA_ONE_CALL (cuMemcpyDtoH) \
|
|
|
|
CUDA_ONE_CALL (cuMemcpyDtoHAsync) \
|
|
|
|
CUDA_ONE_CALL (cuMemcpyHtoD) \
|
|
|
|
CUDA_ONE_CALL (cuMemcpyHtoDAsync) \
|
|
|
|
CUDA_ONE_CALL (cuMemFree) \
|
|
|
|
CUDA_ONE_CALL (cuMemFreeHost) \
|
|
|
|
CUDA_ONE_CALL (cuMemGetAddressRange) \
|
|
|
|
CUDA_ONE_CALL (cuMemHostGetDevicePointer)\
|
|
|
|
CUDA_ONE_CALL (cuModuleGetFunction) \
|
|
|
|
CUDA_ONE_CALL (cuModuleGetGlobal) \
|
|
|
|
CUDA_ONE_CALL (cuModuleLoad) \
|
|
|
|
CUDA_ONE_CALL (cuModuleLoadData) \
|
|
|
|
CUDA_ONE_CALL (cuModuleUnload) \
|
|
|
|
CUDA_ONE_CALL (cuStreamCreate) \
|
|
|
|
CUDA_ONE_CALL (cuStreamDestroy) \
|
|
|
|
CUDA_ONE_CALL (cuStreamQuery) \
|
|
|
|
CUDA_ONE_CALL (cuStreamSynchronize) \
|
|
|
|
CUDA_ONE_CALL (cuStreamWaitEvent)
|
|
|
|
# define CUDA_ONE_CALL(call) \
|
|
|
|
__typeof (call) *call;
|
|
|
|
struct cuda_lib_s {
|
|
|
|
CUDA_CALLS
|
|
|
|
} cuda_lib;
|
|
|
|
|
|
|
|
/* -1 if init_cuda_lib has not been called yet, false
|
|
|
|
if it has been and failed, true if it has been and succeeded. */
|
|
|
|
static signed char cuda_lib_inited = -1;
|
|
|
|
|
|
|
|
/* Dynamically load the CUDA runtime library and initialize function
|
|
|
|
pointers, return false if unsuccessful, true if successful. */
|
|
|
|
static bool
|
|
|
|
init_cuda_lib (void)
|
|
|
|
{
|
|
|
|
if (cuda_lib_inited != -1)
|
|
|
|
return cuda_lib_inited;
|
|
|
|
const char *cuda_runtime_lib = "libcuda.so.1";
|
|
|
|
void *h = dlopen (cuda_runtime_lib, RTLD_LAZY);
|
|
|
|
cuda_lib_inited = false;
|
|
|
|
if (h == NULL)
|
|
|
|
return false;
|
|
|
|
# undef CUDA_ONE_CALL
|
|
|
|
# define CUDA_ONE_CALL(call) CUDA_ONE_CALL_1 (call)
|
|
|
|
# define CUDA_ONE_CALL_1(call) \
|
|
|
|
cuda_lib.call = dlsym (h, #call); \
|
|
|
|
if (cuda_lib.call == NULL) \
|
|
|
|
return false;
|
|
|
|
CUDA_CALLS
|
|
|
|
cuda_lib_inited = true;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
# undef CUDA_ONE_CALL
|
|
|
|
# undef CUDA_ONE_CALL_1
|
|
|
|
# define CUDA_CALL_PREFIX cuda_lib.
|
|
|
|
#else
|
|
|
|
# define CUDA_CALL_PREFIX
|
|
|
|
# define init_cuda_lib() true
|
|
|
|
#endif
|
|
|
|
|
|
|
|
/* Convenience macros for the frequently used CUDA library call and
|
|
|
|
error handling sequence as well as CUDA library calls that
|
|
|
|
do the error checking themselves or don't do it at all. */
|
|
|
|
|
|
|
|
#define CUDA_CALL_ERET(ERET, FN, ...) \
|
|
|
|
do { \
|
|
|
|
unsigned __r \
|
|
|
|
= CUDA_CALL_PREFIX FN (__VA_ARGS__); \
|
|
|
|
if (__r != CUDA_SUCCESS) \
|
|
|
|
{ \
|
|
|
|
GOMP_PLUGIN_error (#FN " error: %s", \
|
|
|
|
cuda_error (__r)); \
|
|
|
|
return ERET; \
|
|
|
|
} \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
#define CUDA_CALL(FN, ...) \
|
|
|
|
CUDA_CALL_ERET (false, FN, __VA_ARGS__)
|
|
|
|
|
|
|
|
#define CUDA_CALL_ASSERT(FN, ...) \
|
|
|
|
do { \
|
|
|
|
unsigned __r \
|
|
|
|
= CUDA_CALL_PREFIX FN (__VA_ARGS__); \
|
|
|
|
if (__r != CUDA_SUCCESS) \
|
|
|
|
{ \
|
|
|
|
GOMP_PLUGIN_fatal (#FN " error: %s", \
|
|
|
|
cuda_error (__r)); \
|
|
|
|
} \
|
|
|
|
} while (0)
|
|
|
|
|
|
|
|
#define CUDA_CALL_NOCHECK(FN, ...) \
|
|
|
|
CUDA_CALL_PREFIX FN (__VA_ARGS__)
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
static const char *
|
2015-08-28 15:33:40 +00:00
|
|
|
cuda_error (CUresult r)
|
|
|
|
{
|
2017-04-10 11:32:00 +00:00
|
|
|
#if CUDA_VERSION < 7000
|
|
|
|
/* Specified in documentation and present in library from at least
|
|
|
|
5.5. Not declared in header file prior to 7.0. */
|
|
|
|
extern CUresult cuGetErrorString (CUresult, const char **);
|
|
|
|
#endif
|
|
|
|
const char *desc;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuGetErrorString, r, &desc);
|
2017-04-10 11:32:00 +00:00
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
desc = "unknown cuda error";
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
return desc;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static unsigned int instantiated_devices = 0;
|
|
|
|
static pthread_mutex_t ptx_dev_lock = PTHREAD_MUTEX_INITIALIZER;
|
|
|
|
|
|
|
|
struct ptx_stream
|
|
|
|
{
|
|
|
|
CUstream stream;
|
|
|
|
pthread_t host_thread;
|
|
|
|
bool multithreaded;
|
|
|
|
|
|
|
|
CUdeviceptr d;
|
|
|
|
void *h;
|
|
|
|
void *h_begin;
|
|
|
|
void *h_end;
|
|
|
|
void *h_next;
|
|
|
|
void *h_prev;
|
|
|
|
void *h_tail;
|
|
|
|
|
|
|
|
struct ptx_stream *next;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Thread-specific data for PTX. */
|
|
|
|
|
|
|
|
struct nvptx_thread
|
|
|
|
{
|
|
|
|
struct ptx_stream *current_stream;
|
|
|
|
struct ptx_device *ptx_dev;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct map
|
|
|
|
{
|
|
|
|
int async;
|
|
|
|
size_t size;
|
|
|
|
char mappings[0];
|
|
|
|
};
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static bool
|
2015-08-28 15:33:40 +00:00
|
|
|
map_init (struct ptx_stream *s)
|
|
|
|
{
|
|
|
|
int size = getpagesize ();
|
|
|
|
|
|
|
|
assert (s);
|
|
|
|
assert (!s->d);
|
|
|
|
assert (!s->h);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemAllocHost, &s->h, size);
|
|
|
|
CUDA_CALL (cuMemHostGetDevicePointer, &s->d, s->h, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
assert (s->h);
|
|
|
|
|
|
|
|
s->h_begin = s->h;
|
|
|
|
s->h_end = s->h_begin + size;
|
|
|
|
s->h_next = s->h_prev = s->h_tail = s->h_begin;
|
|
|
|
|
|
|
|
assert (s->h_next);
|
|
|
|
assert (s->h_end);
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static bool
|
2015-08-28 15:33:40 +00:00
|
|
|
map_fini (struct ptx_stream *s)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemFreeHost, s->h);
|
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
map_pop (struct ptx_stream *s)
|
|
|
|
{
|
|
|
|
struct map *m;
|
|
|
|
|
|
|
|
assert (s != NULL);
|
|
|
|
assert (s->h_next);
|
|
|
|
assert (s->h_prev);
|
|
|
|
assert (s->h_tail);
|
|
|
|
|
|
|
|
m = s->h_tail;
|
|
|
|
|
|
|
|
s->h_tail += m->size;
|
|
|
|
|
|
|
|
if (s->h_tail >= s->h_end)
|
|
|
|
s->h_tail = s->h_begin + (int) (s->h_tail - s->h_end);
|
|
|
|
|
|
|
|
if (s->h_next == s->h_tail)
|
|
|
|
s->h_prev = s->h_next;
|
|
|
|
|
|
|
|
assert (s->h_next >= s->h_begin);
|
|
|
|
assert (s->h_tail >= s->h_begin);
|
|
|
|
assert (s->h_prev >= s->h_begin);
|
|
|
|
|
|
|
|
assert (s->h_next <= s->h_end);
|
|
|
|
assert (s->h_tail <= s->h_end);
|
|
|
|
assert (s->h_prev <= s->h_end);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
map_push (struct ptx_stream *s, int async, size_t size, void **h, void **d)
|
|
|
|
{
|
|
|
|
int left;
|
|
|
|
int offset;
|
|
|
|
struct map *m;
|
|
|
|
|
|
|
|
assert (s != NULL);
|
|
|
|
|
|
|
|
left = s->h_end - s->h_next;
|
|
|
|
size += sizeof (struct map);
|
|
|
|
|
|
|
|
assert (s->h_prev);
|
|
|
|
assert (s->h_next);
|
|
|
|
|
|
|
|
if (size >= left)
|
|
|
|
{
|
|
|
|
m = s->h_prev;
|
|
|
|
m->size += left;
|
|
|
|
s->h_next = s->h_begin;
|
|
|
|
|
|
|
|
if (s->h_next + size > s->h_end)
|
|
|
|
GOMP_PLUGIN_fatal ("unable to push map");
|
|
|
|
}
|
|
|
|
|
|
|
|
assert (s->h_next);
|
|
|
|
|
|
|
|
m = s->h_next;
|
|
|
|
m->async = async;
|
|
|
|
m->size = size;
|
|
|
|
|
|
|
|
offset = (void *)&m->mappings[0] - s->h;
|
|
|
|
|
|
|
|
*d = (void *)(s->d + offset);
|
|
|
|
*h = (void *)(s->h + offset);
|
|
|
|
|
|
|
|
s->h_prev = s->h_next;
|
|
|
|
s->h_next += size;
|
|
|
|
|
|
|
|
assert (s->h_prev);
|
|
|
|
assert (s->h_next);
|
|
|
|
|
|
|
|
assert (s->h_next >= s->h_begin);
|
|
|
|
assert (s->h_tail >= s->h_begin);
|
|
|
|
assert (s->h_prev >= s->h_begin);
|
|
|
|
assert (s->h_next <= s->h_end);
|
|
|
|
assert (s->h_tail <= s->h_end);
|
|
|
|
assert (s->h_prev <= s->h_end);
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
/* Target data function launch information. */
|
|
|
|
|
|
|
|
struct targ_fn_launch
|
|
|
|
{
|
|
|
|
const char *fn;
|
|
|
|
unsigned short dim[GOMP_DIM_MAX];
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Target PTX object information. */
|
|
|
|
|
|
|
|
struct targ_ptx_obj
|
|
|
|
{
|
|
|
|
const char *code;
|
|
|
|
size_t size;
|
|
|
|
};
|
|
|
|
|
|
|
|
/* Target data image information. */
|
|
|
|
|
|
|
|
typedef struct nvptx_tdata
|
|
|
|
{
|
|
|
|
const struct targ_ptx_obj *ptx_objs;
|
|
|
|
unsigned ptx_num;
|
|
|
|
|
|
|
|
const char *const *var_names;
|
|
|
|
unsigned var_num;
|
|
|
|
|
|
|
|
const struct targ_fn_launch *fn_descs;
|
|
|
|
unsigned fn_num;
|
|
|
|
} nvptx_tdata_t;
|
|
|
|
|
|
|
|
/* Descriptor of a loaded function. */
|
|
|
|
|
|
|
|
struct targ_fn_descriptor
|
|
|
|
{
|
|
|
|
CUfunction fn;
|
|
|
|
const struct targ_fn_launch *launch;
|
2017-10-07 00:16:47 +00:00
|
|
|
int regs_per_thread;
|
|
|
|
int max_threads_per_block;
|
2017-04-10 11:32:00 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
/* A loaded PTX image. */
|
|
|
|
struct ptx_image_data
|
|
|
|
{
|
|
|
|
const void *target_data;
|
|
|
|
CUmodule module;
|
|
|
|
|
|
|
|
struct targ_fn_descriptor *fns; /* Array of functions. */
|
|
|
|
|
|
|
|
struct ptx_image_data *next;
|
|
|
|
};
|
|
|
|
|
2015-08-28 15:33:40 +00:00
|
|
|
struct ptx_device
|
|
|
|
{
|
|
|
|
CUcontext ctx;
|
|
|
|
bool ctx_shared;
|
|
|
|
CUdevice dev;
|
|
|
|
struct ptx_stream *null_stream;
|
|
|
|
/* All non-null streams associated with this device (actually context),
|
|
|
|
either created implicitly or passed in from the user (via
|
|
|
|
acc_set_cuda_stream). */
|
|
|
|
struct ptx_stream *active_streams;
|
|
|
|
struct {
|
|
|
|
struct ptx_stream **arr;
|
|
|
|
int size;
|
|
|
|
} async_streams;
|
|
|
|
/* A lock for use when manipulating the above stream list and array. */
|
|
|
|
pthread_mutex_t stream_lock;
|
|
|
|
int ord;
|
|
|
|
bool overlap;
|
|
|
|
bool map;
|
|
|
|
bool concur;
|
|
|
|
bool mkern;
|
2017-10-07 00:16:47 +00:00
|
|
|
int mode;
|
|
|
|
int clock_khz;
|
|
|
|
int num_sms;
|
|
|
|
int regs_per_block;
|
|
|
|
int regs_per_sm;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
struct ptx_image_data *images; /* Images loaded on device. */
|
|
|
|
pthread_mutex_t image_lock; /* Lock for above list. */
|
|
|
|
|
2015-08-28 15:33:40 +00:00
|
|
|
struct ptx_device *next;
|
|
|
|
};
|
|
|
|
|
|
|
|
enum ptx_event_type
|
|
|
|
{
|
|
|
|
PTX_EVT_MEM,
|
|
|
|
PTX_EVT_KNL,
|
|
|
|
PTX_EVT_SYNC,
|
|
|
|
PTX_EVT_ASYNC_CLEANUP
|
|
|
|
};
|
|
|
|
|
|
|
|
struct ptx_event
|
|
|
|
{
|
|
|
|
CUevent *evt;
|
|
|
|
int type;
|
|
|
|
void *addr;
|
|
|
|
int ord;
|
2017-10-07 00:16:47 +00:00
|
|
|
int val;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
struct ptx_event *next;
|
|
|
|
};
|
|
|
|
|
|
|
|
static pthread_mutex_t ptx_event_lock;
|
|
|
|
static struct ptx_event *ptx_events;
|
|
|
|
|
|
|
|
static struct ptx_device **ptx_devices;
|
|
|
|
|
|
|
|
static inline struct nvptx_thread *
|
|
|
|
nvptx_thread (void)
|
|
|
|
{
|
|
|
|
return (struct nvptx_thread *) GOMP_PLUGIN_acc_thread ();
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static bool
|
2015-08-28 15:33:40 +00:00
|
|
|
init_streams_for_device (struct ptx_device *ptx_dev, int concurrency)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
struct ptx_stream *null_stream
|
|
|
|
= GOMP_PLUGIN_malloc (sizeof (struct ptx_stream));
|
|
|
|
|
|
|
|
null_stream->stream = NULL;
|
|
|
|
null_stream->host_thread = pthread_self ();
|
|
|
|
null_stream->multithreaded = true;
|
|
|
|
null_stream->d = (CUdeviceptr) NULL;
|
|
|
|
null_stream->h = NULL;
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!map_init (null_stream))
|
|
|
|
return false;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
ptx_dev->null_stream = null_stream;
|
2015-08-28 15:33:40 +00:00
|
|
|
ptx_dev->active_streams = NULL;
|
|
|
|
pthread_mutex_init (&ptx_dev->stream_lock, NULL);
|
|
|
|
|
|
|
|
if (concurrency < 1)
|
|
|
|
concurrency = 1;
|
|
|
|
|
|
|
|
/* This is just a guess -- make space for as many async streams as the
|
|
|
|
current device is capable of concurrently executing. This can grow
|
|
|
|
later as necessary. No streams are created yet. */
|
|
|
|
ptx_dev->async_streams.arr
|
|
|
|
= GOMP_PLUGIN_malloc (concurrency * sizeof (struct ptx_stream *));
|
|
|
|
ptx_dev->async_streams.size = concurrency;
|
|
|
|
|
|
|
|
for (i = 0; i < concurrency; i++)
|
|
|
|
ptx_dev->async_streams.arr[i] = NULL;
|
2017-10-07 00:16:47 +00:00
|
|
|
|
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static bool
|
2015-08-28 15:33:40 +00:00
|
|
|
fini_streams_for_device (struct ptx_device *ptx_dev)
|
|
|
|
{
|
|
|
|
free (ptx_dev->async_streams.arr);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
bool ret = true;
|
2015-08-28 15:33:40 +00:00
|
|
|
while (ptx_dev->active_streams != NULL)
|
|
|
|
{
|
|
|
|
struct ptx_stream *s = ptx_dev->active_streams;
|
|
|
|
ptx_dev->active_streams = ptx_dev->active_streams->next;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
ret &= map_fini (s);
|
|
|
|
|
|
|
|
CUresult r = CUDA_CALL_NOCHECK (cuStreamDestroy, s->stream);
|
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("cuStreamDestroy error: %s", cuda_error (r));
|
|
|
|
ret = false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
free (s);
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
ret &= map_fini (ptx_dev->null_stream);
|
2015-08-28 15:33:40 +00:00
|
|
|
free (ptx_dev->null_stream);
|
2017-10-07 00:16:47 +00:00
|
|
|
return ret;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Select a stream for (OpenACC-semantics) ASYNC argument for the current
|
|
|
|
thread THREAD (and also current device/context). If CREATE is true, create
|
|
|
|
the stream if it does not exist (or use EXISTING if it is non-NULL), and
|
|
|
|
associate the stream with the same thread argument. Returns stream to use
|
|
|
|
as result. */
|
|
|
|
|
|
|
|
static struct ptx_stream *
|
|
|
|
select_stream_for_async (int async, pthread_t thread, bool create,
|
|
|
|
CUstream existing)
|
|
|
|
{
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
/* Local copy of TLS variable. */
|
|
|
|
struct ptx_device *ptx_dev = nvthd->ptx_dev;
|
|
|
|
struct ptx_stream *stream = NULL;
|
|
|
|
int orig_async = async;
|
|
|
|
|
|
|
|
/* The special value acc_async_noval (-1) maps (for now) to an
|
|
|
|
implicitly-created stream, which is then handled the same as any other
|
|
|
|
numbered async stream. Other options are available, e.g. using the null
|
|
|
|
stream for anonymous async operations, or choosing an idle stream from an
|
|
|
|
active set. But, stick with this for now. */
|
|
|
|
if (async > acc_async_sync)
|
|
|
|
async++;
|
|
|
|
|
|
|
|
if (create)
|
|
|
|
pthread_mutex_lock (&ptx_dev->stream_lock);
|
|
|
|
|
|
|
|
/* NOTE: AFAICT there's no particular need for acc_async_sync to map to the
|
|
|
|
null stream, and in fact better performance may be obtainable if it doesn't
|
|
|
|
(because the null stream enforces overly-strict synchronisation with
|
|
|
|
respect to other streams for legacy reasons, and that's probably not
|
|
|
|
needed with OpenACC). Maybe investigate later. */
|
|
|
|
if (async == acc_async_sync)
|
|
|
|
stream = ptx_dev->null_stream;
|
|
|
|
else if (async >= 0 && async < ptx_dev->async_streams.size
|
|
|
|
&& ptx_dev->async_streams.arr[async] && !(create && existing))
|
|
|
|
stream = ptx_dev->async_streams.arr[async];
|
|
|
|
else if (async >= 0 && create)
|
|
|
|
{
|
|
|
|
if (async >= ptx_dev->async_streams.size)
|
|
|
|
{
|
|
|
|
int i, newsize = ptx_dev->async_streams.size * 2;
|
|
|
|
|
|
|
|
if (async >= newsize)
|
|
|
|
newsize = async + 1;
|
|
|
|
|
|
|
|
ptx_dev->async_streams.arr
|
|
|
|
= GOMP_PLUGIN_realloc (ptx_dev->async_streams.arr,
|
|
|
|
newsize * sizeof (struct ptx_stream *));
|
|
|
|
|
|
|
|
for (i = ptx_dev->async_streams.size; i < newsize; i++)
|
|
|
|
ptx_dev->async_streams.arr[i] = NULL;
|
|
|
|
|
|
|
|
ptx_dev->async_streams.size = newsize;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Create a new stream on-demand if there isn't one already, or if we're
|
|
|
|
setting a particular async value to an existing (externally-provided)
|
|
|
|
stream. */
|
|
|
|
if (!ptx_dev->async_streams.arr[async] || existing)
|
|
|
|
{
|
|
|
|
CUresult r;
|
|
|
|
struct ptx_stream *s
|
|
|
|
= GOMP_PLUGIN_malloc (sizeof (struct ptx_stream));
|
|
|
|
|
|
|
|
if (existing)
|
|
|
|
s->stream = existing;
|
|
|
|
else
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuStreamCreate, &s->stream,
|
|
|
|
CU_STREAM_DEFAULT);
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r != CUDA_SUCCESS)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
pthread_mutex_unlock (&ptx_dev->stream_lock);
|
|
|
|
GOMP_PLUGIN_fatal ("cuStreamCreate error: %s",
|
|
|
|
cuda_error (r));
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* If CREATE is true, we're going to be queueing some work on this
|
|
|
|
stream. Associate it with the current host thread. */
|
|
|
|
s->host_thread = thread;
|
|
|
|
s->multithreaded = false;
|
|
|
|
|
|
|
|
s->d = (CUdeviceptr) NULL;
|
|
|
|
s->h = NULL;
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!map_init (s))
|
|
|
|
{
|
|
|
|
pthread_mutex_unlock (&ptx_dev->stream_lock);
|
|
|
|
GOMP_PLUGIN_fatal ("map_init fail");
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
s->next = ptx_dev->active_streams;
|
|
|
|
ptx_dev->active_streams = s;
|
|
|
|
ptx_dev->async_streams.arr[async] = s;
|
|
|
|
}
|
|
|
|
|
|
|
|
stream = ptx_dev->async_streams.arr[async];
|
|
|
|
}
|
|
|
|
else if (async < 0)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
if (create)
|
|
|
|
pthread_mutex_unlock (&ptx_dev->stream_lock);
|
|
|
|
GOMP_PLUGIN_fatal ("bad async %d", async);
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
if (create)
|
|
|
|
{
|
|
|
|
assert (stream != NULL);
|
|
|
|
|
|
|
|
/* If we're trying to use the same stream from different threads
|
|
|
|
simultaneously, set stream->multithreaded to true. This affects the
|
|
|
|
behaviour of acc_async_test_all and acc_wait_all, which are supposed to
|
|
|
|
only wait for asynchronous launches from the same host thread they are
|
|
|
|
invoked on. If multiple threads use the same async value, we make note
|
|
|
|
of that here and fall back to testing/waiting for all threads in those
|
|
|
|
functions. */
|
|
|
|
if (thread != stream->host_thread)
|
|
|
|
stream->multithreaded = true;
|
|
|
|
|
|
|
|
pthread_mutex_unlock (&ptx_dev->stream_lock);
|
|
|
|
}
|
|
|
|
else if (stream && !stream->multithreaded
|
|
|
|
&& !pthread_equal (stream->host_thread, thread))
|
|
|
|
GOMP_PLUGIN_fatal ("async %d used on wrong thread", orig_async);
|
|
|
|
|
|
|
|
return stream;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Initialize the device. Return TRUE on success, else FALSE. PTX_DEV_LOCK
|
|
|
|
should be locked on entry and remains locked on exit. */
|
2017-04-10 11:32:00 +00:00
|
|
|
|
2015-08-28 15:33:40 +00:00
|
|
|
static bool
|
|
|
|
nvptx_init (void)
|
|
|
|
{
|
|
|
|
int ndevs;
|
|
|
|
|
|
|
|
if (instantiated_devices != 0)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
ptx_events = NULL;
|
|
|
|
pthread_mutex_init (&ptx_event_lock, NULL);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!init_cuda_lib ())
|
|
|
|
return false;
|
|
|
|
|
|
|
|
CUDA_CALL (cuInit, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuDeviceGetCount, &ndevs);
|
2015-08-28 15:33:40 +00:00
|
|
|
ptx_devices = GOMP_PLUGIN_malloc_cleared (sizeof (struct ptx_device *)
|
|
|
|
* ndevs);
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Select the N'th PTX device for the current host thread. The device must
|
|
|
|
have been previously opened before calling this function. */
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static bool
|
2015-08-28 15:33:40 +00:00
|
|
|
nvptx_attach_host_thread_to_device (int n)
|
|
|
|
{
|
|
|
|
CUdevice dev;
|
|
|
|
CUresult r;
|
|
|
|
struct ptx_device *ptx_dev;
|
|
|
|
CUcontext thd_ctx;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuCtxGetDevice, &dev);
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("cuCtxGetDevice error: %s", cuda_error (r));
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
if (r != CUDA_ERROR_INVALID_CONTEXT && dev == n)
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
else
|
|
|
|
{
|
|
|
|
CUcontext old_ctx;
|
|
|
|
|
|
|
|
ptx_dev = ptx_devices[n];
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!ptx_dev)
|
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("device %d not found", n);
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuCtxGetCurrent, &thd_ctx);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
/* We don't necessarily have a current context (e.g. if it has been
|
|
|
|
destroyed. Pop it if we do though. */
|
|
|
|
if (thd_ctx != NULL)
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuCtxPopCurrent, &old_ctx);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuCtxPushCurrent, ptx_dev->ctx);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct ptx_device *
|
|
|
|
nvptx_open_device (int n)
|
|
|
|
{
|
|
|
|
struct ptx_device *ptx_dev;
|
|
|
|
CUdevice dev, ctx_dev;
|
|
|
|
CUresult r;
|
|
|
|
int async_engines, pi;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGet, &dev, n);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
ptx_dev = GOMP_PLUGIN_malloc (sizeof (struct ptx_device));
|
|
|
|
|
|
|
|
ptx_dev->ord = n;
|
|
|
|
ptx_dev->dev = dev;
|
|
|
|
ptx_dev->ctx_shared = false;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuCtxGetDevice, &ctx_dev);
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r != CUDA_SUCCESS && r != CUDA_ERROR_INVALID_CONTEXT)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("cuCtxGetDevice error: %s", cuda_error (r));
|
|
|
|
return NULL;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
if (r != CUDA_ERROR_INVALID_CONTEXT && ctx_dev != dev)
|
|
|
|
{
|
|
|
|
/* The current host thread has an active context for a different device.
|
|
|
|
Detach it. */
|
|
|
|
CUcontext old_ctx;
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuCtxPopCurrent, &old_ctx);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuCtxGetCurrent, &ptx_dev->ctx);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
if (!ptx_dev->ctx)
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuCtxCreate, &ptx_dev->ctx, CU_CTX_SCHED_AUTO, dev);
|
2015-08-28 15:33:40 +00:00
|
|
|
else
|
|
|
|
ptx_dev->ctx_shared = true;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,
|
|
|
|
&pi, CU_DEVICE_ATTRIBUTE_GPU_OVERLAP, dev);
|
2015-08-28 15:33:40 +00:00
|
|
|
ptx_dev->overlap = pi;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,
|
|
|
|
&pi, CU_DEVICE_ATTRIBUTE_CAN_MAP_HOST_MEMORY, dev);
|
2015-08-28 15:33:40 +00:00
|
|
|
ptx_dev->map = pi;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,
|
|
|
|
&pi, CU_DEVICE_ATTRIBUTE_CONCURRENT_KERNELS, dev);
|
2015-08-28 15:33:40 +00:00
|
|
|
ptx_dev->concur = pi;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,
|
|
|
|
&pi, CU_DEVICE_ATTRIBUTE_COMPUTE_MODE, dev);
|
2015-08-28 15:33:40 +00:00
|
|
|
ptx_dev->mode = pi;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,
|
|
|
|
&pi, CU_DEVICE_ATTRIBUTE_INTEGRATED, dev);
|
2015-08-28 15:33:40 +00:00
|
|
|
ptx_dev->mkern = pi;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,
|
|
|
|
&pi, CU_DEVICE_ATTRIBUTE_CLOCK_RATE, dev);
|
|
|
|
ptx_dev->clock_khz = pi;
|
|
|
|
|
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,
|
|
|
|
&pi, CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, dev);
|
|
|
|
ptx_dev->num_sms = pi;
|
|
|
|
|
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,
|
|
|
|
&pi, CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_BLOCK, dev);
|
|
|
|
ptx_dev->regs_per_block = pi;
|
|
|
|
|
|
|
|
/* CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_MULTIPROCESSOR = 82 is defined only
|
|
|
|
in CUDA 6.0 and newer. */
|
|
|
|
r = CUDA_CALL_NOCHECK (cuDeviceGetAttribute, &pi, 82, dev);
|
|
|
|
/* Fallback: use limit of registers per block, which is usually equal. */
|
|
|
|
if (r == CUDA_ERROR_INVALID_VALUE)
|
|
|
|
pi = ptx_dev->regs_per_block;
|
|
|
|
else if (r != CUDA_SUCCESS)
|
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("cuDeviceGetAttribute error: %s", cuda_error (r));
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
ptx_dev->regs_per_sm = pi;
|
|
|
|
|
|
|
|
CUDA_CALL_ERET (NULL, cuDeviceGetAttribute,
|
|
|
|
&pi, CU_DEVICE_ATTRIBUTE_WARP_SIZE, dev);
|
|
|
|
if (pi != 32)
|
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("Only warp size 32 is supported");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
r = CUDA_CALL_NOCHECK (cuDeviceGetAttribute, &async_engines,
|
|
|
|
CU_DEVICE_ATTRIBUTE_ASYNC_ENGINE_COUNT, dev);
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
async_engines = 1;
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
ptx_dev->images = NULL;
|
|
|
|
pthread_mutex_init (&ptx_dev->image_lock, NULL);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!init_streams_for_device (ptx_dev, async_engines))
|
|
|
|
return NULL;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
return ptx_dev;
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static bool
|
2015-08-28 15:33:40 +00:00
|
|
|
nvptx_close_device (struct ptx_device *ptx_dev)
|
|
|
|
{
|
|
|
|
if (!ptx_dev)
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!fini_streams_for_device (ptx_dev))
|
|
|
|
return false;
|
2017-04-10 11:32:00 +00:00
|
|
|
|
|
|
|
pthread_mutex_destroy (&ptx_dev->image_lock);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
if (!ptx_dev->ctx_shared)
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuCtxDestroy, ptx_dev->ctx);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
free (ptx_dev);
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
nvptx_get_num_devices (void)
|
|
|
|
{
|
|
|
|
int n;
|
|
|
|
|
|
|
|
/* PR libgomp/65099: Currently, we only support offloading in 64-bit
|
|
|
|
configurations. */
|
|
|
|
if (sizeof (void *) != 8)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* This function will be called before the plugin has been initialized in
|
|
|
|
order to enumerate available devices, but CUDA API routines can't be used
|
|
|
|
until cuInit has been called. Just call it now (but don't yet do any
|
|
|
|
further initialization). */
|
|
|
|
if (instantiated_devices == 0)
|
2017-04-10 11:32:00 +00:00
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!init_cuda_lib ())
|
|
|
|
return 0;
|
|
|
|
CUresult r = CUDA_CALL_NOCHECK (cuInit, 0);
|
2017-04-10 11:32:00 +00:00
|
|
|
/* This is not an error: e.g. we may have CUDA libraries installed but
|
|
|
|
no devices available. */
|
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
return 0;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (-1, cuDeviceGetCount, &n);
|
2015-08-28 15:33:40 +00:00
|
|
|
return n;
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static bool
|
2017-04-10 11:32:00 +00:00
|
|
|
link_ptx (CUmodule *module, const struct targ_ptx_obj *ptx_objs,
|
|
|
|
unsigned num_objs)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
2017-04-10 11:32:00 +00:00
|
|
|
CUjit_option opts[6];
|
|
|
|
void *optvals[6];
|
2015-08-28 15:33:40 +00:00
|
|
|
float elapsed = 0.0;
|
2017-10-07 00:16:47 +00:00
|
|
|
char elog[1024];
|
|
|
|
char ilog[16384];
|
2015-08-28 15:33:40 +00:00
|
|
|
CUlinkState linkstate;
|
|
|
|
CUresult r;
|
|
|
|
void *linkout;
|
|
|
|
size_t linkoutsize __attribute__ ((unused));
|
|
|
|
|
|
|
|
opts[0] = CU_JIT_WALL_TIME;
|
|
|
|
optvals[0] = &elapsed;
|
|
|
|
|
|
|
|
opts[1] = CU_JIT_INFO_LOG_BUFFER;
|
|
|
|
optvals[1] = &ilog[0];
|
|
|
|
|
|
|
|
opts[2] = CU_JIT_INFO_LOG_BUFFER_SIZE_BYTES;
|
2017-10-07 00:16:47 +00:00
|
|
|
optvals[2] = (void *) sizeof ilog;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
opts[3] = CU_JIT_ERROR_LOG_BUFFER;
|
|
|
|
optvals[3] = &elog[0];
|
|
|
|
|
|
|
|
opts[4] = CU_JIT_ERROR_LOG_BUFFER_SIZE_BYTES;
|
2017-10-07 00:16:47 +00:00
|
|
|
optvals[4] = (void *) sizeof elog;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
opts[5] = CU_JIT_LOG_VERBOSE;
|
|
|
|
optvals[5] = (void *) 1;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuLinkCreate, 6, opts, optvals, &linkstate);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
for (; num_objs--; ptx_objs++)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
2017-04-10 11:32:00 +00:00
|
|
|
/* cuLinkAddData's 'data' argument erroneously omits the const
|
|
|
|
qualifier. */
|
|
|
|
GOMP_PLUGIN_debug (0, "Loading:\n---\n%s\n---\n", ptx_objs->code);
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuLinkAddData, linkstate, CU_JIT_INPUT_PTX,
|
|
|
|
(char *) ptx_objs->code, ptx_objs->size,
|
|
|
|
0, 0, 0, 0);
|
2017-04-10 11:32:00 +00:00
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("Link error log %s\n", &elog[0]);
|
2017-10-07 00:16:47 +00:00
|
|
|
GOMP_PLUGIN_error ("cuLinkAddData (ptx_code) error: %s",
|
2017-04-10 11:32:00 +00:00
|
|
|
cuda_error (r));
|
2017-10-07 00:16:47 +00:00
|
|
|
return false;
|
2017-04-10 11:32:00 +00:00
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
GOMP_PLUGIN_debug (0, "Linking\n");
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuLinkComplete, linkstate, &linkout, &linkoutsize);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
GOMP_PLUGIN_debug (0, "Link complete: %fms\n", elapsed);
|
|
|
|
GOMP_PLUGIN_debug (0, "Link log %s\n", &ilog[0]);
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
if (r != CUDA_SUCCESS)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("cuLinkComplete error: %s", cuda_error (r));
|
|
|
|
return false;
|
|
|
|
}
|
2017-04-10 11:32:00 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuModuleLoadData, module, linkout);
|
|
|
|
CUDA_CALL (cuLinkDestroy, linkstate);
|
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
event_gc (bool memmap_lockable)
|
|
|
|
{
|
|
|
|
struct ptx_event *ptx_event = ptx_events;
|
2017-10-07 00:16:47 +00:00
|
|
|
struct ptx_event *async_cleanups = NULL;
|
2015-08-28 15:33:40 +00:00
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
pthread_mutex_lock (&ptx_event_lock);
|
|
|
|
|
|
|
|
while (ptx_event != NULL)
|
|
|
|
{
|
|
|
|
CUresult r;
|
|
|
|
struct ptx_event *e = ptx_event;
|
|
|
|
|
|
|
|
ptx_event = ptx_event->next;
|
|
|
|
|
|
|
|
if (e->ord != nvthd->ptx_dev->ord)
|
|
|
|
continue;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuEventQuery, *e->evt);
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r == CUDA_SUCCESS)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
bool append_async = false;
|
2015-08-28 15:33:40 +00:00
|
|
|
CUevent *te;
|
|
|
|
|
|
|
|
te = e->evt;
|
|
|
|
|
|
|
|
switch (e->type)
|
|
|
|
{
|
|
|
|
case PTX_EVT_MEM:
|
|
|
|
case PTX_EVT_SYNC:
|
|
|
|
break;
|
|
|
|
|
|
|
|
case PTX_EVT_KNL:
|
|
|
|
map_pop (e->addr);
|
|
|
|
break;
|
|
|
|
|
|
|
|
case PTX_EVT_ASYNC_CLEANUP:
|
|
|
|
{
|
|
|
|
/* The function gomp_plugin_async_unmap_vars needs to claim the
|
|
|
|
memory-map splay tree lock for the current device, so we
|
|
|
|
can't call it when one of our callers has already claimed
|
|
|
|
the lock. In that case, just delay the GC for this event
|
|
|
|
until later. */
|
|
|
|
if (!memmap_lockable)
|
|
|
|
continue;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
append_async = true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_NOCHECK (cuEventDestroy, *te);
|
2015-08-28 15:33:40 +00:00
|
|
|
free ((void *)te);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
/* Unlink 'e' from ptx_events list. */
|
2015-08-28 15:33:40 +00:00
|
|
|
if (ptx_events == e)
|
|
|
|
ptx_events = ptx_events->next;
|
|
|
|
else
|
|
|
|
{
|
|
|
|
struct ptx_event *e_ = ptx_events;
|
|
|
|
while (e_->next != e)
|
|
|
|
e_ = e_->next;
|
|
|
|
e_->next = e_->next->next;
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
if (append_async)
|
|
|
|
{
|
|
|
|
e->next = async_cleanups;
|
|
|
|
async_cleanups = e;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
free (e);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock (&ptx_event_lock);
|
2017-10-07 00:16:47 +00:00
|
|
|
|
|
|
|
/* We have to do these here, after ptx_event_lock is released. */
|
|
|
|
while (async_cleanups)
|
|
|
|
{
|
|
|
|
struct ptx_event *e = async_cleanups;
|
|
|
|
async_cleanups = async_cleanups->next;
|
|
|
|
|
|
|
|
GOMP_PLUGIN_async_unmap_vars (e->addr, e->val);
|
|
|
|
free (e);
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
2017-10-07 00:16:47 +00:00
|
|
|
event_add (enum ptx_event_type type, CUevent *e, void *h, int val)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
struct ptx_event *ptx_event;
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
assert (type == PTX_EVT_MEM || type == PTX_EVT_KNL || type == PTX_EVT_SYNC
|
|
|
|
|| type == PTX_EVT_ASYNC_CLEANUP);
|
|
|
|
|
|
|
|
ptx_event = GOMP_PLUGIN_malloc (sizeof (struct ptx_event));
|
|
|
|
ptx_event->type = type;
|
|
|
|
ptx_event->evt = e;
|
|
|
|
ptx_event->addr = h;
|
|
|
|
ptx_event->ord = nvthd->ptx_dev->ord;
|
2017-10-07 00:16:47 +00:00
|
|
|
ptx_event->val = val;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
pthread_mutex_lock (&ptx_event_lock);
|
|
|
|
|
|
|
|
ptx_event->next = ptx_events;
|
|
|
|
ptx_events = ptx_event;
|
|
|
|
|
|
|
|
pthread_mutex_unlock (&ptx_event_lock);
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static void
|
2015-08-28 15:33:40 +00:00
|
|
|
nvptx_exec (void (*fn), size_t mapnum, void **hostaddrs, void **devaddrs,
|
2017-04-10 11:32:00 +00:00
|
|
|
int async, unsigned *dims, void *targ_mem_desc)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
struct targ_fn_descriptor *targ_fn = (struct targ_fn_descriptor *) fn;
|
|
|
|
CUfunction function;
|
|
|
|
CUresult r;
|
|
|
|
int i;
|
|
|
|
struct ptx_stream *dev_str;
|
|
|
|
void *kargs[1];
|
|
|
|
void *hp, *dp;
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
const char *maybe_abort_msg = "(perhaps abort was called)";
|
|
|
|
|
|
|
|
function = targ_fn->fn;
|
|
|
|
|
|
|
|
dev_str = select_stream_for_async (async, pthread_self (), false, NULL);
|
|
|
|
assert (dev_str == nvthd->current_stream);
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
/* Initialize the launch dimensions. Typically this is constant,
|
|
|
|
provided by the device compiler, but we must permit runtime
|
|
|
|
values. */
|
|
|
|
int seen_zero = 0;
|
|
|
|
for (i = 0; i != GOMP_DIM_MAX; i++)
|
|
|
|
{
|
|
|
|
if (targ_fn->launch->dim[i])
|
|
|
|
dims[i] = targ_fn->launch->dim[i];
|
|
|
|
if (!dims[i])
|
|
|
|
seen_zero = 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (seen_zero)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
/* See if the user provided GOMP_OPENACC_DIM environment
|
|
|
|
variable to specify runtime defaults. */
|
|
|
|
static int default_dims[GOMP_DIM_MAX];
|
|
|
|
|
|
|
|
pthread_mutex_lock (&ptx_dev_lock);
|
|
|
|
if (!default_dims[0])
|
|
|
|
{
|
|
|
|
/* We only read the environment variable once. You can't
|
|
|
|
change it in the middle of execution. The syntax is
|
|
|
|
the same as for the -fopenacc-dim compilation option. */
|
|
|
|
const char *env_var = getenv ("GOMP_OPENACC_DIM");
|
|
|
|
if (env_var)
|
|
|
|
{
|
|
|
|
const char *pos = env_var;
|
|
|
|
|
|
|
|
for (i = 0; *pos && i != GOMP_DIM_MAX; i++)
|
|
|
|
{
|
|
|
|
if (i && *pos++ != ':')
|
|
|
|
break;
|
|
|
|
if (*pos != ':')
|
|
|
|
{
|
|
|
|
const char *eptr;
|
|
|
|
|
|
|
|
errno = 0;
|
|
|
|
long val = strtol (pos, (char **)&eptr, 10);
|
|
|
|
if (errno || val < 0 || (unsigned)val != val)
|
|
|
|
break;
|
|
|
|
default_dims[i] = (int)val;
|
|
|
|
pos = eptr;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
int warp_size, block_size, dev_size, cpu_size;
|
|
|
|
CUdevice dev = nvptx_thread()->ptx_dev->dev;
|
|
|
|
/* 32 is the default for known hardware. */
|
|
|
|
int gang = 0, worker = 32, vector = 32;
|
|
|
|
CUdevice_attribute cu_tpb, cu_ws, cu_mpc, cu_tpm;
|
|
|
|
|
|
|
|
cu_tpb = CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_BLOCK;
|
|
|
|
cu_ws = CU_DEVICE_ATTRIBUTE_WARP_SIZE;
|
|
|
|
cu_mpc = CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT;
|
|
|
|
cu_tpm = CU_DEVICE_ATTRIBUTE_MAX_THREADS_PER_MULTIPROCESSOR;
|
|
|
|
|
|
|
|
if (CUDA_CALL_NOCHECK (cuDeviceGetAttribute, &block_size, cu_tpb,
|
|
|
|
dev) == CUDA_SUCCESS
|
|
|
|
&& CUDA_CALL_NOCHECK (cuDeviceGetAttribute, &warp_size, cu_ws,
|
|
|
|
dev) == CUDA_SUCCESS
|
|
|
|
&& CUDA_CALL_NOCHECK (cuDeviceGetAttribute, &dev_size, cu_mpc,
|
|
|
|
dev) == CUDA_SUCCESS
|
|
|
|
&& CUDA_CALL_NOCHECK (cuDeviceGetAttribute, &cpu_size, cu_tpm,
|
|
|
|
dev) == CUDA_SUCCESS)
|
|
|
|
{
|
|
|
|
GOMP_PLUGIN_debug (0, " warp_size=%d, block_size=%d,"
|
|
|
|
" dev_size=%d, cpu_size=%d\n",
|
|
|
|
warp_size, block_size, dev_size, cpu_size);
|
|
|
|
gang = (cpu_size / block_size) * dev_size;
|
|
|
|
worker = block_size / warp_size;
|
|
|
|
vector = warp_size;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* There is no upper bound on the gang size. The best size
|
|
|
|
matches the hardware configuration. Logical gangs are
|
|
|
|
scheduled onto physical hardware. To maximize usage, we
|
|
|
|
should guess a large number. */
|
|
|
|
if (default_dims[GOMP_DIM_GANG] < 1)
|
|
|
|
default_dims[GOMP_DIM_GANG] = gang ? gang : 1024;
|
|
|
|
/* The worker size must not exceed the hardware. */
|
|
|
|
if (default_dims[GOMP_DIM_WORKER] < 1
|
|
|
|
|| (default_dims[GOMP_DIM_WORKER] > worker && gang))
|
|
|
|
default_dims[GOMP_DIM_WORKER] = worker;
|
|
|
|
/* The vector size must exactly match the hardware. */
|
|
|
|
if (default_dims[GOMP_DIM_VECTOR] < 1
|
|
|
|
|| (default_dims[GOMP_DIM_VECTOR] != vector && gang))
|
|
|
|
default_dims[GOMP_DIM_VECTOR] = vector;
|
|
|
|
|
|
|
|
GOMP_PLUGIN_debug (0, " default dimensions [%d,%d,%d]\n",
|
|
|
|
default_dims[GOMP_DIM_GANG],
|
|
|
|
default_dims[GOMP_DIM_WORKER],
|
|
|
|
default_dims[GOMP_DIM_VECTOR]);
|
|
|
|
}
|
|
|
|
pthread_mutex_unlock (&ptx_dev_lock);
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
for (i = 0; i != GOMP_DIM_MAX; i++)
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!dims[i])
|
|
|
|
dims[i] = default_dims[i];
|
2017-04-10 11:32:00 +00:00
|
|
|
}
|
|
|
|
|
2015-08-28 15:33:40 +00:00
|
|
|
/* This reserves a chunk of a pre-allocated page of memory mapped on both
|
|
|
|
the host and the device. HP is a host pointer to the new chunk, and DP is
|
|
|
|
the corresponding device pointer. */
|
|
|
|
map_push (dev_str, async, mapnum * sizeof (void *), &hp, &dp);
|
|
|
|
|
|
|
|
GOMP_PLUGIN_debug (0, " %s: prepare mappings\n", __FUNCTION__);
|
|
|
|
|
|
|
|
/* Copy the array of arguments to the mapped page. */
|
|
|
|
for (i = 0; i < mapnum; i++)
|
|
|
|
((void **) hp)[i] = devaddrs[i];
|
|
|
|
|
|
|
|
/* Copy the (device) pointers to arguments to the device (dp and hp might in
|
|
|
|
fact have the same value on a unified-memory system). */
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuMemcpy, (CUdeviceptr) dp, (CUdeviceptr) hp,
|
|
|
|
mapnum * sizeof (void *));
|
2017-04-10 11:32:00 +00:00
|
|
|
GOMP_PLUGIN_debug (0, " %s: kernel %s: launch"
|
|
|
|
" gangs=%u, workers=%u, vectors=%u\n",
|
2017-10-07 00:16:47 +00:00
|
|
|
__FUNCTION__, targ_fn->launch->fn, dims[GOMP_DIM_GANG],
|
|
|
|
dims[GOMP_DIM_WORKER], dims[GOMP_DIM_VECTOR]);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
// OpenACC CUDA
|
|
|
|
//
|
2017-04-10 11:32:00 +00:00
|
|
|
// num_gangs nctaid.x
|
|
|
|
// num_workers ntid.y
|
|
|
|
// vector length ntid.x
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
kargs[0] = &dp;
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuLaunchKernel, function,
|
|
|
|
dims[GOMP_DIM_GANG], 1, 1,
|
|
|
|
dims[GOMP_DIM_VECTOR], dims[GOMP_DIM_WORKER], 1,
|
|
|
|
0, dev_str->stream, kargs, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
#ifndef DISABLE_ASYNC
|
|
|
|
if (async < acc_async_noval)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuStreamSynchronize, dev_str->stream);
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r == CUDA_ERROR_LAUNCH_FAILED)
|
|
|
|
GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s %s\n", cuda_error (r),
|
|
|
|
maybe_abort_msg);
|
|
|
|
else if (r != CUDA_SUCCESS)
|
|
|
|
GOMP_PLUGIN_fatal ("cuStreamSynchronize error: %s", cuda_error (r));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
CUevent *e;
|
|
|
|
|
|
|
|
e = (CUevent *)GOMP_PLUGIN_malloc (sizeof (CUevent));
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuEventCreate, e, CU_EVENT_DISABLE_TIMING);
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r == CUDA_ERROR_LAUNCH_FAILED)
|
|
|
|
GOMP_PLUGIN_fatal ("cuEventCreate error: %s %s\n", cuda_error (r),
|
|
|
|
maybe_abort_msg);
|
|
|
|
else if (r != CUDA_SUCCESS)
|
|
|
|
GOMP_PLUGIN_fatal ("cuEventCreate error: %s", cuda_error (r));
|
|
|
|
|
|
|
|
event_gc (true);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuEventRecord, *e, dev_str->stream);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
event_add (PTX_EVT_KNL, e, (void *)dev_str, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
#else
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuCtxSynchronize, );
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r == CUDA_ERROR_LAUNCH_FAILED)
|
|
|
|
GOMP_PLUGIN_fatal ("cuCtxSynchronize error: %s %s\n", cuda_error (r),
|
|
|
|
maybe_abort_msg);
|
|
|
|
else if (r != CUDA_SUCCESS)
|
|
|
|
GOMP_PLUGIN_fatal ("cuCtxSynchronize error: %s", cuda_error (r));
|
|
|
|
#endif
|
|
|
|
|
|
|
|
GOMP_PLUGIN_debug (0, " %s: kernel %s: finished\n", __FUNCTION__,
|
2017-04-10 11:32:00 +00:00
|
|
|
targ_fn->launch->fn);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
#ifndef DISABLE_ASYNC
|
|
|
|
if (async < acc_async_noval)
|
|
|
|
#endif
|
|
|
|
map_pop (dev_str);
|
|
|
|
}
|
|
|
|
|
|
|
|
void * openacc_get_current_cuda_context (void);
|
|
|
|
|
|
|
|
static void *
|
|
|
|
nvptx_alloc (size_t s)
|
|
|
|
{
|
|
|
|
CUdeviceptr d;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (NULL, cuMemAlloc, &d, s);
|
|
|
|
return (void *) d;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static bool
|
2015-08-28 15:33:40 +00:00
|
|
|
nvptx_free (void *p)
|
|
|
|
{
|
|
|
|
CUdeviceptr pb;
|
|
|
|
size_t ps;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemGetAddressRange, &pb, &ps, (CUdeviceptr) p);
|
|
|
|
if ((CUdeviceptr) p != pb)
|
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid device address");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemFree, (CUdeviceptr) p);
|
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
|
|
|
|
static bool
|
2015-08-28 15:33:40 +00:00
|
|
|
nvptx_host2dev (void *d, const void *h, size_t s)
|
|
|
|
{
|
|
|
|
CUdeviceptr pb;
|
|
|
|
size_t ps;
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
if (!s)
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
if (!d)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid device address");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemGetAddressRange, &pb, &ps, (CUdeviceptr) d);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
if (!pb)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid device address");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
if (!h)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid host address");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
if (d == h)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid host or device address");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
if ((void *)(d + s) > (void *)(pb + ps))
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid size");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
#ifndef DISABLE_ASYNC
|
2017-10-07 00:16:47 +00:00
|
|
|
if (nvthd && nvthd->current_stream != nvthd->ptx_dev->null_stream)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
CUevent *e = (CUevent *)GOMP_PLUGIN_malloc (sizeof (CUevent));
|
|
|
|
CUDA_CALL (cuEventCreate, e, CU_EVENT_DISABLE_TIMING);
|
2015-08-28 15:33:40 +00:00
|
|
|
event_gc (false);
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemcpyHtoDAsync,
|
|
|
|
(CUdeviceptr) d, h, s, nvthd->current_stream->stream);
|
|
|
|
CUDA_CALL (cuEventRecord, *e, nvthd->current_stream->stream);
|
|
|
|
event_add (PTX_EVT_MEM, e, (void *)h, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
#endif
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemcpyHtoD, (CUdeviceptr) d, h, s);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
static bool
|
2015-08-28 15:33:40 +00:00
|
|
|
nvptx_dev2host (void *h, const void *d, size_t s)
|
|
|
|
{
|
|
|
|
CUdeviceptr pb;
|
|
|
|
size_t ps;
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
if (!s)
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
if (!d)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid device address");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemGetAddressRange, &pb, &ps, (CUdeviceptr) d);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
if (!pb)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid device address");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
if (!h)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid host address");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
if (d == h)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid host or device address");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
if ((void *)(d + s) > (void *)(pb + ps))
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("invalid size");
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
#ifndef DISABLE_ASYNC
|
2017-10-07 00:16:47 +00:00
|
|
|
if (nvthd && nvthd->current_stream != nvthd->ptx_dev->null_stream)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
CUevent *e = (CUevent *) GOMP_PLUGIN_malloc (sizeof (CUevent));
|
|
|
|
CUDA_CALL (cuEventCreate, e, CU_EVENT_DISABLE_TIMING);
|
2015-08-28 15:33:40 +00:00
|
|
|
event_gc (false);
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemcpyDtoHAsync,
|
|
|
|
h, (CUdeviceptr) d, s, nvthd->current_stream->stream);
|
|
|
|
CUDA_CALL (cuEventRecord, *e, nvthd->current_stream->stream);
|
|
|
|
event_add (PTX_EVT_MEM, e, (void *)h, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
#endif
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL (cuMemcpyDtoH, h, (CUdeviceptr) d, s);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvptx_set_async (int async)
|
|
|
|
{
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
nvthd->current_stream
|
|
|
|
= select_stream_for_async (async, pthread_self (), true, NULL);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
nvptx_async_test (int async)
|
|
|
|
{
|
|
|
|
CUresult r;
|
|
|
|
struct ptx_stream *s;
|
|
|
|
|
|
|
|
s = select_stream_for_async (async, pthread_self (), false, NULL);
|
|
|
|
|
|
|
|
if (!s)
|
|
|
|
GOMP_PLUGIN_fatal ("unknown async %d", async);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuStreamQuery, s->stream);
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r == CUDA_SUCCESS)
|
|
|
|
{
|
|
|
|
/* The oacc-parallel.c:goacc_wait function calls this hook to determine
|
|
|
|
whether all work has completed on this stream, and if so omits the call
|
|
|
|
to the wait hook. If that happens, event_gc might not get called
|
|
|
|
(which prevents variables from getting unmapped and their associated
|
|
|
|
device storage freed), so call it here. */
|
|
|
|
event_gc (true);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
else if (r == CUDA_ERROR_NOT_READY)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
GOMP_PLUGIN_fatal ("cuStreamQuery error: %s", cuda_error (r));
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
nvptx_async_test_all (void)
|
|
|
|
{
|
|
|
|
struct ptx_stream *s;
|
|
|
|
pthread_t self = pthread_self ();
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
pthread_mutex_lock (&nvthd->ptx_dev->stream_lock);
|
|
|
|
|
|
|
|
for (s = nvthd->ptx_dev->active_streams; s != NULL; s = s->next)
|
|
|
|
{
|
|
|
|
if ((s->multithreaded || pthread_equal (s->host_thread, self))
|
2017-10-07 00:16:47 +00:00
|
|
|
&& CUDA_CALL_NOCHECK (cuStreamQuery,
|
|
|
|
s->stream) == CUDA_ERROR_NOT_READY)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock);
|
|
|
|
|
|
|
|
event_gc (true);
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvptx_wait (int async)
|
|
|
|
{
|
|
|
|
struct ptx_stream *s;
|
|
|
|
|
|
|
|
s = select_stream_for_async (async, pthread_self (), false, NULL);
|
|
|
|
if (!s)
|
|
|
|
GOMP_PLUGIN_fatal ("unknown async %d", async);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuStreamSynchronize, s->stream);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
event_gc (true);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvptx_wait_async (int async1, int async2)
|
|
|
|
{
|
|
|
|
CUevent *e;
|
|
|
|
struct ptx_stream *s1, *s2;
|
|
|
|
pthread_t self = pthread_self ();
|
|
|
|
|
|
|
|
/* The stream that is waiting (rather than being waited for) doesn't
|
|
|
|
necessarily have to exist already. */
|
|
|
|
s2 = select_stream_for_async (async2, self, true, NULL);
|
|
|
|
|
|
|
|
s1 = select_stream_for_async (async1, self, false, NULL);
|
|
|
|
if (!s1)
|
|
|
|
GOMP_PLUGIN_fatal ("invalid async 1\n");
|
|
|
|
|
|
|
|
if (s1 == s2)
|
|
|
|
GOMP_PLUGIN_fatal ("identical parameters");
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
e = (CUevent *) GOMP_PLUGIN_malloc (sizeof (CUevent));
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuEventCreate, e, CU_EVENT_DISABLE_TIMING);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
event_gc (true);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuEventRecord, *e, s1->stream);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
event_add (PTX_EVT_SYNC, e, NULL, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuStreamWaitEvent, s2->stream, *e, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvptx_wait_all (void)
|
|
|
|
{
|
|
|
|
CUresult r;
|
|
|
|
struct ptx_stream *s;
|
|
|
|
pthread_t self = pthread_self ();
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
pthread_mutex_lock (&nvthd->ptx_dev->stream_lock);
|
|
|
|
|
|
|
|
/* Wait for active streams initiated by this thread (or by multiple threads)
|
|
|
|
to complete. */
|
|
|
|
for (s = nvthd->ptx_dev->active_streams; s != NULL; s = s->next)
|
|
|
|
{
|
|
|
|
if (s->multithreaded || pthread_equal (s->host_thread, self))
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
r = CUDA_CALL_NOCHECK (cuStreamQuery, s->stream);
|
2015-08-28 15:33:40 +00:00
|
|
|
if (r == CUDA_SUCCESS)
|
|
|
|
continue;
|
|
|
|
else if (r != CUDA_ERROR_NOT_READY)
|
|
|
|
GOMP_PLUGIN_fatal ("cuStreamQuery error: %s", cuda_error (r));
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuStreamSynchronize, s->stream);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock);
|
|
|
|
|
|
|
|
event_gc (true);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvptx_wait_all_async (int async)
|
|
|
|
{
|
|
|
|
struct ptx_stream *waiting_stream, *other_stream;
|
|
|
|
CUevent *e;
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
pthread_t self = pthread_self ();
|
|
|
|
|
|
|
|
/* The stream doing the waiting. This could be the first mention of the
|
|
|
|
stream, so create it if necessary. */
|
|
|
|
waiting_stream
|
|
|
|
= select_stream_for_async (async, pthread_self (), true, NULL);
|
|
|
|
|
|
|
|
/* Launches on the null stream already block on other streams in the
|
|
|
|
context. */
|
|
|
|
if (!waiting_stream || waiting_stream == nvthd->ptx_dev->null_stream)
|
|
|
|
return;
|
|
|
|
|
|
|
|
event_gc (true);
|
|
|
|
|
|
|
|
pthread_mutex_lock (&nvthd->ptx_dev->stream_lock);
|
|
|
|
|
|
|
|
for (other_stream = nvthd->ptx_dev->active_streams;
|
|
|
|
other_stream != NULL;
|
|
|
|
other_stream = other_stream->next)
|
|
|
|
{
|
|
|
|
if (!other_stream->multithreaded
|
|
|
|
&& !pthread_equal (other_stream->host_thread, self))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
e = (CUevent *) GOMP_PLUGIN_malloc (sizeof (CUevent));
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuEventCreate, e, CU_EVENT_DISABLE_TIMING);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
/* Record an event on the waited-for stream. */
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuEventRecord, *e, other_stream->stream);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
event_add (PTX_EVT_SYNC, e, NULL, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuStreamWaitEvent, waiting_stream->stream, *e, 0);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *
|
|
|
|
nvptx_get_current_cuda_device (void)
|
|
|
|
{
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
if (!nvthd || !nvthd->ptx_dev)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return &nvthd->ptx_dev->dev;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *
|
|
|
|
nvptx_get_current_cuda_context (void)
|
|
|
|
{
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
if (!nvthd || !nvthd->ptx_dev)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return nvthd->ptx_dev->ctx;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *
|
|
|
|
nvptx_get_cuda_stream (int async)
|
|
|
|
{
|
|
|
|
struct ptx_stream *s;
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
if (!nvthd || !nvthd->ptx_dev)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
s = select_stream_for_async (async, pthread_self (), false, NULL);
|
|
|
|
|
|
|
|
return s ? s->stream : NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int
|
|
|
|
nvptx_set_cuda_stream (int async, void *stream)
|
|
|
|
{
|
|
|
|
struct ptx_stream *oldstream;
|
|
|
|
pthread_t self = pthread_self ();
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
|
|
|
|
|
|
|
if (async < 0)
|
|
|
|
GOMP_PLUGIN_fatal ("bad async %d", async);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
pthread_mutex_lock (&nvthd->ptx_dev->stream_lock);
|
|
|
|
|
2015-08-28 15:33:40 +00:00
|
|
|
/* We have a list of active streams and an array mapping async values to
|
|
|
|
entries of that list. We need to take "ownership" of the passed-in stream,
|
|
|
|
and add it to our list, removing the previous entry also (if there was one)
|
|
|
|
in order to prevent resource leaks. Note the potential for surprise
|
|
|
|
here: maybe we should keep track of passed-in streams and leave it up to
|
|
|
|
the user to tidy those up, but that doesn't work for stream handles
|
|
|
|
returned from acc_get_cuda_stream above... */
|
|
|
|
|
|
|
|
oldstream = select_stream_for_async (async, self, false, NULL);
|
|
|
|
|
|
|
|
if (oldstream)
|
|
|
|
{
|
|
|
|
if (nvthd->ptx_dev->active_streams == oldstream)
|
|
|
|
nvthd->ptx_dev->active_streams = nvthd->ptx_dev->active_streams->next;
|
|
|
|
else
|
|
|
|
{
|
|
|
|
struct ptx_stream *s = nvthd->ptx_dev->active_streams;
|
|
|
|
while (s->next != oldstream)
|
|
|
|
s = s->next;
|
|
|
|
s->next = s->next->next;
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuStreamDestroy, oldstream->stream);
|
|
|
|
|
|
|
|
if (!map_fini (oldstream))
|
|
|
|
GOMP_PLUGIN_fatal ("error when freeing host memory");
|
|
|
|
|
2015-08-28 15:33:40 +00:00
|
|
|
free (oldstream);
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock (&nvthd->ptx_dev->stream_lock);
|
|
|
|
|
|
|
|
(void) select_stream_for_async (async, self, true, (CUstream) stream);
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Plugin entry points. */
|
|
|
|
|
|
|
|
const char *
|
|
|
|
GOMP_OFFLOAD_get_name (void)
|
|
|
|
{
|
|
|
|
return "nvptx";
|
|
|
|
}
|
|
|
|
|
|
|
|
unsigned int
|
|
|
|
GOMP_OFFLOAD_get_caps (void)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
return GOMP_OFFLOAD_CAP_OPENACC_200 | GOMP_OFFLOAD_CAP_OPENMP_400;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
GOMP_OFFLOAD_get_type (void)
|
|
|
|
{
|
|
|
|
return OFFLOAD_TARGET_TYPE_NVIDIA_PTX;
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
GOMP_OFFLOAD_get_num_devices (void)
|
|
|
|
{
|
|
|
|
return nvptx_get_num_devices ();
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
bool
|
2015-08-28 15:33:40 +00:00
|
|
|
GOMP_OFFLOAD_init_device (int n)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
struct ptx_device *dev;
|
|
|
|
|
2015-08-28 15:33:40 +00:00
|
|
|
pthread_mutex_lock (&ptx_dev_lock);
|
|
|
|
|
|
|
|
if (!nvptx_init () || ptx_devices[n] != NULL)
|
|
|
|
{
|
|
|
|
pthread_mutex_unlock (&ptx_dev_lock);
|
2017-10-07 00:16:47 +00:00
|
|
|
return false;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
dev = nvptx_open_device (n);
|
|
|
|
if (dev)
|
|
|
|
{
|
|
|
|
ptx_devices[n] = dev;
|
|
|
|
instantiated_devices++;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
pthread_mutex_unlock (&ptx_dev_lock);
|
2017-10-07 00:16:47 +00:00
|
|
|
|
|
|
|
return dev != NULL;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
bool
|
2015-08-28 15:33:40 +00:00
|
|
|
GOMP_OFFLOAD_fini_device (int n)
|
|
|
|
{
|
|
|
|
pthread_mutex_lock (&ptx_dev_lock);
|
|
|
|
|
|
|
|
if (ptx_devices[n] != NULL)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!nvptx_attach_host_thread_to_device (n)
|
|
|
|
|| !nvptx_close_device (ptx_devices[n]))
|
|
|
|
{
|
|
|
|
pthread_mutex_unlock (&ptx_dev_lock);
|
|
|
|
return false;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
ptx_devices[n] = NULL;
|
|
|
|
instantiated_devices--;
|
|
|
|
}
|
|
|
|
|
|
|
|
pthread_mutex_unlock (&ptx_dev_lock);
|
2017-10-07 00:16:47 +00:00
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
/* Return the libgomp version number we're compatible with. There is
|
|
|
|
no requirement for cross-version compatibility. */
|
|
|
|
|
|
|
|
unsigned
|
|
|
|
GOMP_OFFLOAD_version (void)
|
|
|
|
{
|
|
|
|
return GOMP_VERSION;
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
/* Initialize __nvptx_clocktick, if present in MODULE. */
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvptx_set_clocktick (CUmodule module, struct ptx_device *dev)
|
|
|
|
{
|
|
|
|
CUdeviceptr dptr;
|
|
|
|
CUresult r = CUDA_CALL_NOCHECK (cuModuleGetGlobal, &dptr, NULL,
|
|
|
|
module, "__nvptx_clocktick");
|
|
|
|
if (r == CUDA_ERROR_NOT_FOUND)
|
|
|
|
return;
|
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
GOMP_PLUGIN_fatal ("cuModuleGetGlobal error: %s", cuda_error (r));
|
|
|
|
double __nvptx_clocktick = 1e-3 / dev->clock_khz;
|
|
|
|
r = CUDA_CALL_NOCHECK (cuMemcpyHtoD, dptr, &__nvptx_clocktick,
|
|
|
|
sizeof (__nvptx_clocktick));
|
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
GOMP_PLUGIN_fatal ("cuMemcpyHtoD error: %s", cuda_error (r));
|
|
|
|
}
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
/* Load the (partial) program described by TARGET_DATA to device
|
|
|
|
number ORD. Allocate and return TARGET_TABLE. */
|
|
|
|
|
2015-08-28 15:33:40 +00:00
|
|
|
int
|
2017-04-10 11:32:00 +00:00
|
|
|
GOMP_OFFLOAD_load_image (int ord, unsigned version, const void *target_data,
|
2015-08-28 15:33:40 +00:00
|
|
|
struct addr_pair **target_table)
|
|
|
|
{
|
|
|
|
CUmodule module;
|
2017-04-10 11:32:00 +00:00
|
|
|
const char *const *var_names;
|
|
|
|
const struct targ_fn_launch *fn_descs;
|
2015-08-28 15:33:40 +00:00
|
|
|
unsigned int fn_entries, var_entries, i, j;
|
|
|
|
struct targ_fn_descriptor *targ_fns;
|
2017-04-10 11:32:00 +00:00
|
|
|
struct addr_pair *targ_tbl;
|
|
|
|
const nvptx_tdata_t *img_header = (const nvptx_tdata_t *) target_data;
|
2015-08-28 15:33:40 +00:00
|
|
|
struct ptx_image_data *new_image;
|
2017-04-10 11:32:00 +00:00
|
|
|
struct ptx_device *dev;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
if (GOMP_VERSION_DEV (version) > GOMP_VERSION_NVIDIA_PTX)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("Offload data incompatible with PTX plugin"
|
|
|
|
" (expected %u, received %u)",
|
|
|
|
GOMP_VERSION_NVIDIA_PTX, GOMP_VERSION_DEV (version));
|
|
|
|
return -1;
|
|
|
|
}
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!nvptx_attach_host_thread_to_device (ord)
|
|
|
|
|| !link_ptx (&module, img_header->ptx_objs, img_header->ptx_num))
|
|
|
|
return -1;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
dev = ptx_devices[ord];
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
/* The mkoffload utility emits a struct of pointers/integers at the
|
|
|
|
start of each offload image. The array of kernel names and the
|
|
|
|
functions addresses form a one-to-one correspondence. */
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
var_entries = img_header->var_num;
|
|
|
|
var_names = img_header->var_names;
|
|
|
|
fn_entries = img_header->fn_num;
|
|
|
|
fn_descs = img_header->fn_descs;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
targ_tbl = GOMP_PLUGIN_malloc (sizeof (struct addr_pair)
|
|
|
|
* (fn_entries + var_entries));
|
|
|
|
targ_fns = GOMP_PLUGIN_malloc (sizeof (struct targ_fn_descriptor)
|
|
|
|
* fn_entries);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
*target_table = targ_tbl;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
new_image = GOMP_PLUGIN_malloc (sizeof (struct ptx_image_data));
|
|
|
|
new_image->target_data = target_data;
|
|
|
|
new_image->module = module;
|
|
|
|
new_image->fns = targ_fns;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
pthread_mutex_lock (&dev->image_lock);
|
|
|
|
new_image->next = dev->images;
|
|
|
|
dev->images = new_image;
|
|
|
|
pthread_mutex_unlock (&dev->image_lock);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
for (i = 0; i < fn_entries; i++, targ_fns++, targ_tbl++)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
CUfunction function;
|
2017-10-07 00:16:47 +00:00
|
|
|
int nregs, mthrs;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (-1, cuModuleGetFunction, &function, module,
|
|
|
|
fn_descs[i].fn);
|
|
|
|
CUDA_CALL_ERET (-1, cuFuncGetAttribute, &nregs,
|
|
|
|
CU_FUNC_ATTRIBUTE_NUM_REGS, function);
|
|
|
|
CUDA_CALL_ERET (-1, cuFuncGetAttribute, &mthrs,
|
|
|
|
CU_FUNC_ATTRIBUTE_MAX_THREADS_PER_BLOCK, function);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
targ_fns->fn = function;
|
|
|
|
targ_fns->launch = &fn_descs[i];
|
2017-10-07 00:16:47 +00:00
|
|
|
targ_fns->regs_per_thread = nregs;
|
|
|
|
targ_fns->max_threads_per_block = mthrs;
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
targ_tbl->start = (uintptr_t) targ_fns;
|
|
|
|
targ_tbl->end = targ_tbl->start + 1;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
for (j = 0; j < var_entries; j++, targ_tbl++)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
CUdeviceptr var;
|
|
|
|
size_t bytes;
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ERET (-1, cuModuleGetGlobal,
|
|
|
|
&var, &bytes, module, var_names[j]);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
targ_tbl->start = (uintptr_t) var;
|
|
|
|
targ_tbl->end = targ_tbl->start + bytes;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
nvptx_set_clocktick (module, dev);
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
return fn_entries + var_entries;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
/* Unload the program described by TARGET_DATA. DEV_DATA is the
|
|
|
|
function descriptors allocated by G_O_load_image. */
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
bool
|
2017-04-10 11:32:00 +00:00
|
|
|
GOMP_OFFLOAD_unload_image (int ord, unsigned version, const void *target_data)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
2017-04-10 11:32:00 +00:00
|
|
|
struct ptx_image_data *image, **prev_p;
|
|
|
|
struct ptx_device *dev = ptx_devices[ord];
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-04-10 11:32:00 +00:00
|
|
|
if (GOMP_VERSION_DEV (version) > GOMP_VERSION_NVIDIA_PTX)
|
2017-10-07 00:16:47 +00:00
|
|
|
{
|
|
|
|
GOMP_PLUGIN_error ("Offload data incompatible with PTX plugin"
|
|
|
|
" (expected %u, received %u)",
|
|
|
|
GOMP_VERSION_NVIDIA_PTX, GOMP_VERSION_DEV (version));
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
bool ret = true;
|
2017-04-10 11:32:00 +00:00
|
|
|
pthread_mutex_lock (&dev->image_lock);
|
|
|
|
for (prev_p = &dev->images; (image = *prev_p) != 0; prev_p = &image->next)
|
|
|
|
if (image->target_data == target_data)
|
|
|
|
{
|
|
|
|
*prev_p = image->next;
|
2017-10-07 00:16:47 +00:00
|
|
|
if (CUDA_CALL_NOCHECK (cuModuleUnload, image->module) != CUDA_SUCCESS)
|
|
|
|
ret = false;
|
2017-04-10 11:32:00 +00:00
|
|
|
free (image->fns);
|
|
|
|
free (image);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
pthread_mutex_unlock (&dev->image_lock);
|
2017-10-07 00:16:47 +00:00
|
|
|
return ret;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void *
|
|
|
|
GOMP_OFFLOAD_alloc (int ord, size_t size)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
if (!nvptx_attach_host_thread_to_device (ord))
|
|
|
|
return NULL;
|
2015-08-28 15:33:40 +00:00
|
|
|
return nvptx_alloc (size);
|
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
bool
|
2015-08-28 15:33:40 +00:00
|
|
|
GOMP_OFFLOAD_free (int ord, void *ptr)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
return (nvptx_attach_host_thread_to_device (ord)
|
|
|
|
&& nvptx_free (ptr));
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
bool
|
2015-08-28 15:33:40 +00:00
|
|
|
GOMP_OFFLOAD_dev2host (int ord, void *dst, const void *src, size_t n)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
return (nvptx_attach_host_thread_to_device (ord)
|
|
|
|
&& nvptx_dev2host (dst, src, n));
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
bool
|
2015-08-28 15:33:40 +00:00
|
|
|
GOMP_OFFLOAD_host2dev (int ord, void *dst, const void *src, size_t n)
|
|
|
|
{
|
2017-10-07 00:16:47 +00:00
|
|
|
return (nvptx_attach_host_thread_to_device (ord)
|
|
|
|
&& nvptx_host2dev (dst, src, n));
|
|
|
|
}
|
|
|
|
|
|
|
|
bool
|
|
|
|
GOMP_OFFLOAD_dev2dev (int ord, void *dst, const void *src, size_t n)
|
|
|
|
{
|
|
|
|
struct ptx_device *ptx_dev = ptx_devices[ord];
|
|
|
|
CUDA_CALL (cuMemcpyDtoDAsync, (CUdeviceptr) dst, (CUdeviceptr) src, n,
|
|
|
|
ptx_dev->null_stream->stream);
|
|
|
|
return true;
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void (*device_run) (int n, void *fn_ptr, void *vars) = NULL;
|
|
|
|
|
|
|
|
void
|
2017-10-07 00:16:47 +00:00
|
|
|
GOMP_OFFLOAD_openacc_exec (void (*fn) (void *), size_t mapnum,
|
|
|
|
void **hostaddrs, void **devaddrs,
|
|
|
|
int async, unsigned *dims, void *targ_mem_desc)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
2017-04-10 11:32:00 +00:00
|
|
|
nvptx_exec (fn, mapnum, hostaddrs, devaddrs, async, dims, targ_mem_desc);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void
|
2017-10-07 00:16:47 +00:00
|
|
|
GOMP_OFFLOAD_openacc_register_async_cleanup (void *targ_mem_desc, int async)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
struct nvptx_thread *nvthd = nvptx_thread ();
|
2017-10-07 00:16:47 +00:00
|
|
|
CUevent *e = (CUevent *) GOMP_PLUGIN_malloc (sizeof (CUevent));
|
2015-08-28 15:33:40 +00:00
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuEventCreate, e, CU_EVENT_DISABLE_TIMING);
|
|
|
|
CUDA_CALL_ASSERT (cuEventRecord, *e, nvthd->current_stream->stream);
|
|
|
|
event_add (PTX_EVT_ASYNC_CLEANUP, e, targ_mem_desc, async);
|
2015-08-28 15:33:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
GOMP_OFFLOAD_openacc_async_test (int async)
|
|
|
|
{
|
|
|
|
return nvptx_async_test (async);
|
|
|
|
}
|
|
|
|
|
|
|
|
int
|
|
|
|
GOMP_OFFLOAD_openacc_async_test_all (void)
|
|
|
|
{
|
|
|
|
return nvptx_async_test_all ();
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
GOMP_OFFLOAD_openacc_async_wait (int async)
|
|
|
|
{
|
|
|
|
nvptx_wait (async);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
GOMP_OFFLOAD_openacc_async_wait_async (int async1, int async2)
|
|
|
|
{
|
|
|
|
nvptx_wait_async (async1, async2);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
GOMP_OFFLOAD_openacc_async_wait_all (void)
|
|
|
|
{
|
|
|
|
nvptx_wait_all ();
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
GOMP_OFFLOAD_openacc_async_wait_all_async (int async)
|
|
|
|
{
|
|
|
|
nvptx_wait_all_async (async);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
GOMP_OFFLOAD_openacc_async_set_async (int async)
|
|
|
|
{
|
|
|
|
nvptx_set_async (async);
|
|
|
|
}
|
|
|
|
|
|
|
|
void *
|
|
|
|
GOMP_OFFLOAD_openacc_create_thread_data (int ord)
|
|
|
|
{
|
|
|
|
struct ptx_device *ptx_dev;
|
|
|
|
struct nvptx_thread *nvthd
|
|
|
|
= GOMP_PLUGIN_malloc (sizeof (struct nvptx_thread));
|
|
|
|
CUcontext thd_ctx;
|
|
|
|
|
|
|
|
ptx_dev = ptx_devices[ord];
|
|
|
|
|
|
|
|
assert (ptx_dev);
|
|
|
|
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuCtxGetCurrent, &thd_ctx);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
assert (ptx_dev->ctx);
|
|
|
|
|
|
|
|
if (!thd_ctx)
|
2017-10-07 00:16:47 +00:00
|
|
|
CUDA_CALL_ASSERT (cuCtxPushCurrent, ptx_dev->ctx);
|
2015-08-28 15:33:40 +00:00
|
|
|
|
|
|
|
nvthd->current_stream = ptx_dev->null_stream;
|
|
|
|
nvthd->ptx_dev = ptx_dev;
|
|
|
|
|
|
|
|
return (void *) nvthd;
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
GOMP_OFFLOAD_openacc_destroy_thread_data (void *data)
|
|
|
|
{
|
|
|
|
free (data);
|
|
|
|
}
|
|
|
|
|
|
|
|
void *
|
2017-10-07 00:16:47 +00:00
|
|
|
GOMP_OFFLOAD_openacc_cuda_get_current_device (void)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
return nvptx_get_current_cuda_device ();
|
|
|
|
}
|
|
|
|
|
|
|
|
void *
|
2017-10-07 00:16:47 +00:00
|
|
|
GOMP_OFFLOAD_openacc_cuda_get_current_context (void)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
return nvptx_get_current_cuda_context ();
|
|
|
|
}
|
|
|
|
|
|
|
|
/* NOTE: This returns a CUstream, not a ptx_stream pointer. */
|
|
|
|
|
|
|
|
void *
|
2017-10-07 00:16:47 +00:00
|
|
|
GOMP_OFFLOAD_openacc_cuda_get_stream (int async)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
return nvptx_get_cuda_stream (async);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* NOTE: This takes a CUstream, not a ptx_stream pointer. */
|
|
|
|
|
|
|
|
int
|
2017-10-07 00:16:47 +00:00
|
|
|
GOMP_OFFLOAD_openacc_cuda_set_stream (int async, void *stream)
|
2015-08-28 15:33:40 +00:00
|
|
|
{
|
|
|
|
return nvptx_set_cuda_stream (async, stream);
|
|
|
|
}
|
2017-10-07 00:16:47 +00:00
|
|
|
|
|
|
|
/* Adjust launch dimensions: pick good values for number of blocks and warps
|
|
|
|
and ensure that number of warps does not exceed CUDA limits as well as GCC's
|
|
|
|
own limits. */
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvptx_adjust_launch_bounds (struct targ_fn_descriptor *fn,
|
|
|
|
struct ptx_device *ptx_dev,
|
|
|
|
int *teams_p, int *threads_p)
|
|
|
|
{
|
|
|
|
int max_warps_block = fn->max_threads_per_block / 32;
|
|
|
|
/* Maximum 32 warps per block is an implementation limit in NVPTX backend
|
|
|
|
and libgcc, which matches documented limit of all GPUs as of 2015. */
|
|
|
|
if (max_warps_block > 32)
|
|
|
|
max_warps_block = 32;
|
|
|
|
if (*threads_p <= 0)
|
|
|
|
*threads_p = 8;
|
|
|
|
if (*threads_p > max_warps_block)
|
|
|
|
*threads_p = max_warps_block;
|
|
|
|
|
|
|
|
int regs_per_block = fn->regs_per_thread * 32 * *threads_p;
|
|
|
|
/* This is an estimate of how many blocks the device can host simultaneously.
|
|
|
|
Actual limit, which may be lower, can be queried with "occupancy control"
|
|
|
|
driver interface (since CUDA 6.0). */
|
|
|
|
int max_blocks = ptx_dev->regs_per_sm / regs_per_block * ptx_dev->num_sms;
|
|
|
|
if (*teams_p <= 0 || *teams_p > max_blocks)
|
|
|
|
*teams_p = max_blocks;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Return the size of per-warp stacks (see gcc -msoft-stack) to use for OpenMP
|
|
|
|
target regions. */
|
|
|
|
|
|
|
|
static size_t
|
|
|
|
nvptx_stacks_size ()
|
|
|
|
{
|
|
|
|
return 128 * 1024;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Return contiguous storage for NUM stacks, each SIZE bytes. */
|
|
|
|
|
|
|
|
static void *
|
|
|
|
nvptx_stacks_alloc (size_t size, int num)
|
|
|
|
{
|
|
|
|
CUdeviceptr stacks;
|
|
|
|
CUresult r = CUDA_CALL_NOCHECK (cuMemAlloc, &stacks, size * num);
|
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
GOMP_PLUGIN_fatal ("cuMemAlloc error: %s", cuda_error (r));
|
|
|
|
return (void *) stacks;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Release storage previously allocated by nvptx_stacks_alloc. */
|
|
|
|
|
|
|
|
static void
|
|
|
|
nvptx_stacks_free (void *p, int num)
|
|
|
|
{
|
|
|
|
CUresult r = CUDA_CALL_NOCHECK (cuMemFree, (CUdeviceptr) p);
|
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
GOMP_PLUGIN_fatal ("cuMemFree error: %s", cuda_error (r));
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
GOMP_OFFLOAD_run (int ord, void *tgt_fn, void *tgt_vars, void **args)
|
|
|
|
{
|
|
|
|
CUfunction function = ((struct targ_fn_descriptor *) tgt_fn)->fn;
|
|
|
|
CUresult r;
|
|
|
|
struct ptx_device *ptx_dev = ptx_devices[ord];
|
|
|
|
const char *maybe_abort_msg = "(perhaps abort was called)";
|
|
|
|
int teams = 0, threads = 0;
|
|
|
|
|
|
|
|
if (!args)
|
|
|
|
GOMP_PLUGIN_fatal ("No target arguments provided");
|
|
|
|
while (*args)
|
|
|
|
{
|
|
|
|
intptr_t id = (intptr_t) *args++, val;
|
|
|
|
if (id & GOMP_TARGET_ARG_SUBSEQUENT_PARAM)
|
|
|
|
val = (intptr_t) *args++;
|
|
|
|
else
|
|
|
|
val = id >> GOMP_TARGET_ARG_VALUE_SHIFT;
|
|
|
|
if ((id & GOMP_TARGET_ARG_DEVICE_MASK) != GOMP_TARGET_ARG_DEVICE_ALL)
|
|
|
|
continue;
|
|
|
|
val = val > INT_MAX ? INT_MAX : val;
|
|
|
|
id &= GOMP_TARGET_ARG_ID_MASK;
|
|
|
|
if (id == GOMP_TARGET_ARG_NUM_TEAMS)
|
|
|
|
teams = val;
|
|
|
|
else if (id == GOMP_TARGET_ARG_THREAD_LIMIT)
|
|
|
|
threads = val;
|
|
|
|
}
|
|
|
|
nvptx_adjust_launch_bounds (tgt_fn, ptx_dev, &teams, &threads);
|
|
|
|
|
|
|
|
size_t stack_size = nvptx_stacks_size ();
|
|
|
|
void *stacks = nvptx_stacks_alloc (stack_size, teams * threads);
|
|
|
|
void *fn_args[] = {tgt_vars, stacks, (void *) stack_size};
|
|
|
|
size_t fn_args_size = sizeof fn_args;
|
|
|
|
void *config[] = {
|
|
|
|
CU_LAUNCH_PARAM_BUFFER_POINTER, fn_args,
|
|
|
|
CU_LAUNCH_PARAM_BUFFER_SIZE, &fn_args_size,
|
|
|
|
CU_LAUNCH_PARAM_END
|
|
|
|
};
|
|
|
|
r = CUDA_CALL_NOCHECK (cuLaunchKernel, function, teams, 1, 1,
|
|
|
|
32, threads, 1, 0, ptx_dev->null_stream->stream,
|
|
|
|
NULL, config);
|
|
|
|
if (r != CUDA_SUCCESS)
|
|
|
|
GOMP_PLUGIN_fatal ("cuLaunchKernel error: %s", cuda_error (r));
|
|
|
|
|
|
|
|
r = CUDA_CALL_NOCHECK (cuCtxSynchronize, );
|
|
|
|
if (r == CUDA_ERROR_LAUNCH_FAILED)
|
|
|
|
GOMP_PLUGIN_fatal ("cuCtxSynchronize error: %s %s\n", cuda_error (r),
|
|
|
|
maybe_abort_msg);
|
|
|
|
else if (r != CUDA_SUCCESS)
|
|
|
|
GOMP_PLUGIN_fatal ("cuCtxSynchronize error: %s", cuda_error (r));
|
|
|
|
nvptx_stacks_free (stacks, teams * threads);
|
|
|
|
}
|
|
|
|
|
|
|
|
void
|
|
|
|
GOMP_OFFLOAD_async_run (int ord, void *tgt_fn, void *tgt_vars, void **args,
|
|
|
|
void *async_data)
|
|
|
|
{
|
|
|
|
GOMP_PLUGIN_fatal ("GOMP_OFFLOAD_async_run unimplemented");
|
|
|
|
}
|