diff options
Diffstat (limited to 'src/gallium/frontends/clover')
69 files changed, 15449 insertions, 0 deletions
diff --git a/src/gallium/frontends/clover/Doxyfile b/src/gallium/frontends/clover/Doxyfile new file mode 100644 index 00000000000..19337bbd656 --- /dev/null +++ b/src/gallium/frontends/clover/Doxyfile @@ -0,0 +1,1716 @@ +# Doxyfile 1.7.4 + +# This file describes the settings to be used by the documentation system +# doxygen (www.doxygen.org) for a project. +# +# All text after a hash (#) is considered a comment and will be ignored. +# The format is: +# TAG = value [value, ...] +# For lists items can also be appended using: +# TAG += value [value, ...] +# Values that contain spaces should be placed between quotes (" "). + +#--------------------------------------------------------------------------- +# Project related configuration options +#--------------------------------------------------------------------------- + +# This tag specifies the encoding used for all characters in the config file +# that follow. The default is UTF-8 which is also the encoding used for all +# text before the first occurrence of this tag. Doxygen uses libiconv (or the +# iconv built into libc) for the transcoding. See +# http://www.gnu.org/software/libiconv for the list of possible encodings. + +DOXYFILE_ENCODING = UTF-8 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded +# by quotes) that should identify the project. + +PROJECT_NAME = Clover + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. +# This could be handy for archiving the generated documentation or +# if some version control system is used. + +PROJECT_NUMBER = + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer +# a quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = + +# With the PROJECT_LOGO tag one can specify an logo or icon that is +# included in the documentation. The maximum height of the logo should not +# exceed 55 pixels and the maximum width should not exceed 200 pixels. +# Doxygen will copy the logo to the output directory. + +PROJECT_LOGO = + +# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) +# base path where the generated documentation will be put. +# If a relative path is entered, it will be relative to the location +# where doxygen was started. If left blank the current directory will be used. + +OUTPUT_DIRECTORY = + +# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create +# 4096 sub-directories (in 2 levels) under the output directory of each output +# format and will distribute the generated files over these directories. +# Enabling this option can be useful when feeding doxygen a huge amount of +# source files, where putting all generated files in the same directory would +# otherwise cause performance problems for the file system. + +CREATE_SUBDIRS = NO + +# The OUTPUT_LANGUAGE tag is used to specify the language in which all +# documentation generated by doxygen is written. Doxygen will use this +# information to generate all constant output in the proper language. +# The default language is English, other supported languages are: +# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional, +# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German, +# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English +# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian, +# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak, +# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese. + +OUTPUT_LANGUAGE = English + +# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will +# include brief member descriptions after the members that are listed in +# the file and class documentation (similar to JavaDoc). +# Set to NO to disable this. + +BRIEF_MEMBER_DESC = YES + +# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend +# the brief description of a member or function before the detailed description. +# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the +# brief descriptions will be completely suppressed. + +REPEAT_BRIEF = YES + +# This tag implements a quasi-intelligent brief description abbreviator +# that is used to form the text in various listings. Each string +# in this list, if found as the leading text of the brief description, will be +# stripped from the text and the result after processing the whole list, is +# used as the annotated text. Otherwise, the brief description is used as-is. +# If left blank, the following values are used ("$name" is automatically +# replaced with the name of the entity): "The $name class" "The $name widget" +# "The $name file" "is" "provides" "specifies" "contains" +# "represents" "a" "an" "the" + +ABBREVIATE_BRIEF = + +# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then +# Doxygen will generate a detailed section even if there is only a brief +# description. + +ALWAYS_DETAILED_SEC = NO + +# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all +# inherited members of a class in the documentation of that class as if those +# members were ordinary class members. Constructors, destructors and assignment +# operators of the base classes will not be shown. + +INLINE_INHERITED_MEMB = NO + +# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full +# path before files name in the file list and in the header files. If set +# to NO the shortest path that makes the file name unique will be used. + +FULL_PATH_NAMES = YES + +# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag +# can be used to strip a user-defined part of the path. Stripping is +# only done if one of the specified strings matches the left-hand part of +# the path. The tag can be used to show relative paths in the file list. +# If left blank the directory from which doxygen is run is used as the +# path to strip. + +STRIP_FROM_PATH = + +# The STRIP_FROM_INC_PATH tag can be used to strip a user-defined part of +# the path mentioned in the documentation of a class, which tells +# the reader which header file to include in order to use a class. +# If left blank only the name of the header file containing the class +# definition is used. Otherwise one should specify the include paths that +# are normally passed to the compiler using the -I flag. + +STRIP_FROM_INC_PATH = + +# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter +# (but less readable) file names. This can be useful if your file system +# doesn't support long names like on DOS, Mac, or CD-ROM. + +SHORT_NAMES = NO + +# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen +# will interpret the first line (until the first dot) of a JavaDoc-style +# comment as the brief description. If set to NO, the JavaDoc +# comments will behave just like regular Qt-style comments +# (thus requiring an explicit @brief command for a brief description.) + +JAVADOC_AUTOBRIEF = YES + +# If the QT_AUTOBRIEF tag is set to YES then Doxygen will +# interpret the first line (until the first dot) of a Qt-style +# comment as the brief description. If set to NO, the comments +# will behave just like regular Qt-style comments (thus requiring +# an explicit \brief command for a brief description.) + +QT_AUTOBRIEF = NO + +# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen +# treat a multi-line C++ special comment block (i.e. a block of //! or /// +# comments) as a brief description. This used to be the default behaviour. +# The new default is to treat a multi-line C++ comment block as a detailed +# description. Set this tag to YES if you prefer the old behaviour instead. + +MULTILINE_CPP_IS_BRIEF = NO + +# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented +# member inherits the documentation from any documented member that it +# re-implements. + +INHERIT_DOCS = YES + +# If the SEPARATE_MEMBER_PAGES tag is set to YES, then doxygen will produce +# a new page for each member. If set to NO, the documentation of a member will +# be part of the file/class/namespace that contains it. + +SEPARATE_MEMBER_PAGES = NO + +# The TAB_SIZE tag can be used to set the number of spaces in a tab. +# Doxygen uses this value to replace tabs by spaces in code fragments. + +TAB_SIZE = 8 + +# This tag can be used to specify a number of aliases that acts +# as commands in the documentation. An alias has the form "name=value". +# For example adding "sideeffect=\par Side Effects:\n" will allow you to +# put the command \sideeffect (or @sideeffect) in the documentation, which +# will result in a user-defined paragraph with heading "Side Effects:". +# You can put \n's in the value part of an alias to insert newlines. + +ALIASES = + +# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C +# sources only. Doxygen will then generate output that is more tailored for C. +# For instance, some of the names that are used will be different. The list +# of all members will be omitted, etc. + +OPTIMIZE_OUTPUT_FOR_C = NO + +# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java +# sources only. Doxygen will then generate output that is more tailored for +# Java. For instance, namespaces will be presented as packages, qualified +# scopes will look different, etc. + +OPTIMIZE_OUTPUT_JAVA = NO + +# Set the OPTIMIZE_FOR_FORTRAN tag to YES if your project consists of Fortran +# sources only. Doxygen will then generate output that is more tailored for +# Fortran. + +OPTIMIZE_FOR_FORTRAN = NO + +# Set the OPTIMIZE_OUTPUT_VHDL tag to YES if your project consists of VHDL +# sources. Doxygen will then generate output that is tailored for +# VHDL. + +OPTIMIZE_OUTPUT_VHDL = NO + +# Doxygen selects the parser to use depending on the extension of the files it +# parses. With this tag you can assign which parser to use for a given extension. +# Doxygen has a built-in mapping, but you can override or extend it using this +# tag. The format is ext=language, where ext is a file extension, and language +# is one of the parsers supported by doxygen: IDL, Java, Javascript, CSharp, C, +# C++, D, PHP, Objective-C, Python, Fortran, VHDL, C, C++. For instance to make +# doxygen treat .inc files as Fortran files (default is PHP), and .f files as C +# (default is Fortran), use: inc=Fortran f=C. Note that for custom extensions +# you also need to set FILE_PATTERNS otherwise the files are not read by doxygen. + +EXTENSION_MAPPING = + +# If you use STL classes (i.e. std::string, std::vector, etc.) but do not want +# to include (a tag file for) the STL sources as input, then you should +# set this tag to YES in order to let doxygen match functions declarations and +# definitions whose arguments contain STL classes (e.g. func(std::string); v.s. +# func(std::string) {}). This also makes the inheritance and collaboration +# diagrams that involve STL classes more complete and accurate. + +BUILTIN_STL_SUPPORT = YES + +# If you use Microsoft's C++/CLI language, you should set this option to YES to +# enable parsing support. + +CPP_CLI_SUPPORT = NO + +# Set the SIP_SUPPORT tag to YES if your project consists of sip sources only. +# Doxygen will parse them like normal C++ but will assume all classes use public +# instead of private inheritance when no explicit protection keyword is present. + +SIP_SUPPORT = NO + +# For Microsoft's IDL there are propget and propput attributes to indicate getter +# and setter methods for a property. Setting this option to YES (the default) +# will make doxygen replace the get and set methods by a property in the +# documentation. This will only work if the methods are indeed getting or +# setting a simple type. If this is not the case, or you want to show the +# methods anyway, you should set this option to NO. + +IDL_PROPERTY_SUPPORT = YES + +# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC +# tag is set to YES, then doxygen will reuse the documentation of the first +# member in the group (if any) for the other members of the group. By default +# all members of a group must be documented explicitly. + +DISTRIBUTE_GROUP_DOC = NO + +# Set the SUBGROUPING tag to YES (the default) to allow class member groups of +# the same type (for instance a group of public functions) to be put as a +# subgroup of that type (e.g. under the Public Functions section). Set it to +# NO to prevent subgrouping. Alternatively, this can be done per class using +# the \nosubgrouping command. + +SUBGROUPING = YES + +# When the INLINE_GROUPED_CLASSES tag is set to YES, classes, structs and +# unions are shown inside the group in which they are included (e.g. using +# @ingroup) instead of on a separate page (for HTML and Man pages) or +# section (for LaTeX and RTF). + +INLINE_GROUPED_CLASSES = NO + +# When TYPEDEF_HIDES_STRUCT is enabled, a typedef of a struct, union, or enum +# is documented as struct, union, or enum with the name of the typedef. So +# typedef struct TypeS {} TypeT, will appear in the documentation as a struct +# with name TypeT. When disabled the typedef will appear as a member of a file, +# namespace, or class. And the struct will be named TypeS. This can typically +# be useful for C code in case the coding convention dictates that all compound +# types are typedef'ed and only the typedef is referenced, never the tag name. + +TYPEDEF_HIDES_STRUCT = NO + +# The SYMBOL_CACHE_SIZE determines the size of the internal cache use to +# determine which symbols to keep in memory and which to flush to disk. +# When the cache is full, less often used symbols will be written to disk. +# For small to medium size projects (<1000 input files) the default value is +# probably good enough. For larger projects a too small cache size can cause +# doxygen to be busy swapping symbols to and from disk most of the time +# causing a significant performance penalty. +# If the system has enough physical memory increasing the cache will improve the +# performance by keeping more symbols in memory. Note that the value works on +# a logarithmic scale so increasing the size by one will roughly double the +# memory usage. The cache size is given by this formula: +# 2^(16+SYMBOL_CACHE_SIZE). The valid range is 0..9, the default is 0, +# corresponding to a cache size of 2^16 = 65536 symbols + +SYMBOL_CACHE_SIZE = 0 + +#--------------------------------------------------------------------------- +# Build related configuration options +#--------------------------------------------------------------------------- + +# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in +# documentation are documented, even if no documentation was available. +# Private class members and static file members will be hidden unless +# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES + +EXTRACT_ALL = NO + +# If the EXTRACT_PRIVATE tag is set to YES all private members of a class +# will be included in the documentation. + +EXTRACT_PRIVATE = NO + +# If the EXTRACT_STATIC tag is set to YES all static members of a file +# will be included in the documentation. + +EXTRACT_STATIC = NO + +# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) +# defined locally in source files will be included in the documentation. +# If set to NO only classes defined in header files are included. + +EXTRACT_LOCAL_CLASSES = YES + +# This flag is only useful for Objective-C code. When set to YES local +# methods, which are defined in the implementation section but not in +# the interface are included in the documentation. +# If set to NO (the default) only methods in the interface are included. + +EXTRACT_LOCAL_METHODS = NO + +# If this flag is set to YES, the members of anonymous namespaces will be +# extracted and appear in the documentation as a namespace called +# 'anonymous_namespace{file}', where file will be replaced with the base +# name of the file that contains the anonymous namespace. By default +# anonymous namespaces are hidden. + +EXTRACT_ANON_NSPACES = YES + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all +# undocumented members of documented classes, files or namespaces. +# If set to NO (the default) these members will be included in the +# various overviews, but no documentation section is generated. +# This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. +# If set to NO (the default) these classes will be included in the various +# overviews. This option has no effect if EXTRACT_ALL is enabled. + +HIDE_UNDOC_CLASSES = NO + +# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all +# friend (class|struct|union) declarations. +# If set to NO (the default) these declarations will be included in the +# documentation. + +HIDE_FRIEND_COMPOUNDS = NO + +# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any +# documentation blocks found inside the body of a function. +# If set to NO (the default) these blocks will be appended to the +# function's detailed documentation block. + +HIDE_IN_BODY_DOCS = NO + +# The INTERNAL_DOCS tag determines if documentation +# that is typed after a \internal command is included. If the tag is set +# to NO (the default) then the documentation will be excluded. +# Set it to YES to include the internal documentation. + +INTERNAL_DOCS = NO + +# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate +# file names in lower-case letters. If set to YES upper-case letters are also +# allowed. This is useful if you have classes or files whose names only differ +# in case and if your file system supports case sensitive file names. Windows +# and Mac users are advised to set this option to NO. + +CASE_SENSE_NAMES = YES + +# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen +# will show members with their full class and namespace scopes in the +# documentation. If set to YES the scope will be hidden. + +HIDE_SCOPE_NAMES = NO + +# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen +# will put a list of the files that are included by a file in the documentation +# of that file. + +SHOW_INCLUDE_FILES = YES + +# If the FORCE_LOCAL_INCLUDES tag is set to YES then Doxygen +# will list include files with double quotes in the documentation +# rather than with sharp brackets. + +FORCE_LOCAL_INCLUDES = NO + +# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] +# is inserted in the documentation for inline members. + +INLINE_INFO = YES + +# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen +# will sort the (detailed) documentation of file and class members +# alphabetically by member name. If set to NO the members will appear in +# declaration order. + +SORT_MEMBER_DOCS = YES + +# If the SORT_BRIEF_DOCS tag is set to YES then doxygen will sort the +# brief documentation of file, namespace and class members alphabetically +# by member name. If set to NO (the default) the members will appear in +# declaration order. + +SORT_BRIEF_DOCS = NO + +# If the SORT_MEMBERS_CTORS_1ST tag is set to YES then doxygen +# will sort the (brief and detailed) documentation of class members so that +# constructors and destructors are listed first. If set to NO (the default) +# the constructors will appear in the respective orders defined by +# SORT_MEMBER_DOCS and SORT_BRIEF_DOCS. +# This tag will be ignored for brief docs if SORT_BRIEF_DOCS is set to NO +# and ignored for detailed docs if SORT_MEMBER_DOCS is set to NO. + +SORT_MEMBERS_CTORS_1ST = NO + +# If the SORT_GROUP_NAMES tag is set to YES then doxygen will sort the +# hierarchy of group names into alphabetical order. If set to NO (the default) +# the group names will appear in their defined order. + +SORT_GROUP_NAMES = NO + +# If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be +# sorted by fully-qualified names, including namespaces. If set to +# NO (the default), the class list will be sorted only by class name, +# not including the namespace part. +# Note: This option is not very useful if HIDE_SCOPE_NAMES is set to YES. +# Note: This option applies only to the class list, not to the +# alphabetical list. + +SORT_BY_SCOPE_NAME = NO + +# If the STRICT_PROTO_MATCHING option is enabled and doxygen fails to +# do proper type resolution of all parameters of a function it will reject a +# match between the prototype and the implementation of a member function even +# if there is only one candidate or it is obvious which candidate to choose +# by doing a simple string match. By disabling STRICT_PROTO_MATCHING doxygen +# will still accept a match between prototype and implementation in such cases. + +STRICT_PROTO_MATCHING = NO + +# The GENERATE_TODOLIST tag can be used to enable (YES) or +# disable (NO) the todo list. This list is created by putting \todo +# commands in the documentation. + +GENERATE_TODOLIST = YES + +# The GENERATE_TESTLIST tag can be used to enable (YES) or +# disable (NO) the test list. This list is created by putting \test +# commands in the documentation. + +GENERATE_TESTLIST = YES + +# The GENERATE_BUGLIST tag can be used to enable (YES) or +# disable (NO) the bug list. This list is created by putting \bug +# commands in the documentation. + +GENERATE_BUGLIST = YES + +# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or +# disable (NO) the deprecated list. This list is created by putting +# \deprecated commands in the documentation. + +GENERATE_DEPRECATEDLIST= YES + +# The ENABLED_SECTIONS tag can be used to enable conditional +# documentation sections, marked by \if sectionname ... \endif. + +ENABLED_SECTIONS = + +# The MAX_INITIALIZER_LINES tag determines the maximum number of lines +# the initial value of a variable or macro consists of for it to appear in +# the documentation. If the initializer consists of more lines than specified +# here it will be hidden. Use a value of 0 to hide initializers completely. +# The appearance of the initializer of individual variables and macros in the +# documentation can be controlled using \showinitializer or \hideinitializer +# command in the documentation regardless of this setting. + +MAX_INITIALIZER_LINES = 30 + +# Set the SHOW_USED_FILES tag to NO to disable the list of files generated +# at the bottom of the documentation of classes and structs. If set to YES the +# list will mention the files that were used to generate the documentation. + +SHOW_USED_FILES = YES + +# If the sources in your project are distributed over multiple directories +# then setting the SHOW_DIRECTORIES tag to YES will show the directory hierarchy +# in the documentation. The default is NO. + +SHOW_DIRECTORIES = NO + +# Set the SHOW_FILES tag to NO to disable the generation of the Files page. +# This will remove the Files entry from the Quick Index and from the +# Folder Tree View (if specified). The default is YES. + +SHOW_FILES = YES + +# Set the SHOW_NAMESPACES tag to NO to disable the generation of the +# Namespaces page. +# This will remove the Namespaces entry from the Quick Index +# and from the Folder Tree View (if specified). The default is YES. + +SHOW_NAMESPACES = YES + +# The FILE_VERSION_FILTER tag can be used to specify a program or script that +# doxygen should invoke to get the current version for each file (typically from +# the version control system). Doxygen will invoke the program by executing (via +# popen()) the command <command> <input-file>, where <command> is the value of +# the FILE_VERSION_FILTER tag, and <input-file> is the name of an input file +# provided by doxygen. Whatever the program writes to standard output +# is used as the file version. See the manual for examples. + +FILE_VERSION_FILTER = + +# The LAYOUT_FILE tag can be used to specify a layout file which will be parsed +# by doxygen. The layout file controls the global structure of the generated +# output files in an output format independent way. The create the layout file +# that represents doxygen's defaults, run doxygen with the -l option. +# You can optionally specify a file name after the option, if omitted +# DoxygenLayout.xml will be used as the name of the layout file. + +LAYOUT_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to warning and progress messages +#--------------------------------------------------------------------------- + +# The QUIET tag can be used to turn on/off the messages that are generated +# by doxygen. Possible values are YES and NO. If left blank NO is used. + +QUIET = NO + +# The WARNINGS tag can be used to turn on/off the warning messages that are +# generated by doxygen. Possible values are YES and NO. If left blank +# NO is used. + +WARNINGS = YES + +# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings +# for undocumented members. If EXTRACT_ALL is set to YES then this flag will +# automatically be disabled. + +WARN_IF_UNDOCUMENTED = NO + +# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for +# potential errors in the documentation, such as not documenting some +# parameters in a documented function, or documenting parameters that +# don't exist or using markup commands wrongly. + +WARN_IF_DOC_ERROR = YES + +# The WARN_NO_PARAMDOC option can be enabled to get warnings for +# functions that are documented, but have no documentation for their parameters +# or return value. If set to NO (the default) doxygen will only warn about +# wrong or incomplete parameter documentation, but not about the absence of +# documentation. + +WARN_NO_PARAMDOC = NO + +# The WARN_FORMAT tag determines the format of the warning messages that +# doxygen can produce. The string should contain the $file, $line, and $text +# tags, which will be replaced by the file and line number from which the +# warning originated and the warning text. Optionally the format may contain +# $version, which will be replaced by the version of the file (if it could +# be obtained via FILE_VERSION_FILTER) + +WARN_FORMAT = "$file:$line: $text" + +# The WARN_LOGFILE tag can be used to specify a file to which warning +# and error messages should be written. If left blank the output is written +# to stderr. + +WARN_LOGFILE = + +#--------------------------------------------------------------------------- +# configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag can be used to specify the files and/or directories that contain +# documented source files. You may enter file names like "myfile.cpp" or +# directories like "/usr/src/myproject". Separate the files or directories +# with spaces. + +INPUT = api/ core/ util/ + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding, which is +# also the default input encoding. Doxygen uses libiconv (or the iconv built +# into libc) for the transcoding. See http://www.gnu.org/software/libiconv for +# the list of possible encodings. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank the following patterns are tested: +# *.c *.cc *.cxx *.cpp *.c++ *.d *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh +# *.hxx *.hpp *.h++ *.idl *.odl *.cs *.php *.php3 *.inc *.m *.mm *.dox *.py +# *.f90 *.f *.for *.vhd *.vhdl + +FILE_PATTERNS = + +# The RECURSIVE tag can be used to turn specify whether or not subdirectories +# should be searched for input files as well. Possible values are YES and NO. +# If left blank NO is used. + +RECURSIVE = NO + +# The EXCLUDE tag can be used to specify files and/or directories that should +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. Note that the wildcards are matched +# against the file with absolute path, so to exclude all test directories +# for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test + +EXCLUDE_SYMBOLS = + +# The EXAMPLE_PATH tag can be used to specify one or more files or +# directories that contain example code fragments that are included (see +# the \include command). + +EXAMPLE_PATH = + +# If the value of the EXAMPLE_PATH tag contains directories, you can use the +# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp +# and *.h) to filter out the source-files in the directories. If left +# blank all files are included. + +EXAMPLE_PATTERNS = + +# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be +# searched for input files to be used with the \include or \dontinclude +# commands irrespective of the value of the RECURSIVE tag. +# Possible values are YES and NO. If left blank NO is used. + +EXAMPLE_RECURSIVE = NO + +# The IMAGE_PATH tag can be used to specify one or more files or +# directories that contain image that are included in the documentation (see +# the \image command). + +IMAGE_PATH = + +# The INPUT_FILTER tag can be used to specify a program that doxygen should +# invoke to filter for each input file. Doxygen will invoke the filter program +# by executing (via popen()) the command <filter> <input-file>, where <filter> +# is the value of the INPUT_FILTER tag, and <input-file> is the name of an +# input file. Doxygen will then use the output that the filter program writes +# to standard output. +# If FILTER_PATTERNS is specified, this tag will be +# ignored. + +INPUT_FILTER = + +# The FILTER_PATTERNS tag can be used to specify filters on a per file pattern +# basis. +# Doxygen will compare the file name with each pattern and apply the +# filter if there is a match. +# The filters are a list of the form: +# pattern=filter (like *.cpp=my_cpp_filter). See INPUT_FILTER for further +# info on how filters are used. If FILTER_PATTERNS is empty or if +# non of the patterns match the file name, INPUT_FILTER is applied. + +FILTER_PATTERNS = + +# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using +# INPUT_FILTER) will be used to filter the input files when producing source +# files to browse (i.e. when SOURCE_BROWSER is set to YES). + +FILTER_SOURCE_FILES = NO + +# The FILTER_SOURCE_PATTERNS tag can be used to specify source filters per file +# pattern. A pattern will override the setting for FILTER_PATTERN (if any) +# and it is also possible to disable source filtering for a specific pattern +# using *.ext= (so without naming a filter). This option only has effect when +# FILTER_SOURCE_FILES is enabled. + +FILTER_SOURCE_PATTERNS = + +#--------------------------------------------------------------------------- +# configuration options related to source browsing +#--------------------------------------------------------------------------- + +# If the SOURCE_BROWSER tag is set to YES then a list of source files will +# be generated. Documented entities will be cross-referenced with these sources. +# Note: To get rid of all source code in the generated output, make sure also +# VERBATIM_HEADERS is set to NO. + +SOURCE_BROWSER = NO + +# Setting the INLINE_SOURCES tag to YES will include the body +# of functions and classes directly in the documentation. + +INLINE_SOURCES = NO + +# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct +# doxygen to hide any special comment blocks from generated source code +# fragments. Normal C and C++ comments will always remain visible. + +STRIP_CODE_COMMENTS = YES + +# If the REFERENCED_BY_RELATION tag is set to YES +# then for each documented function all documented +# functions referencing it will be listed. + +REFERENCED_BY_RELATION = NO + +# If the REFERENCES_RELATION tag is set to YES +# then for each documented function all documented entities +# called/used by that function will be listed. + +REFERENCES_RELATION = NO + +# If the REFERENCES_LINK_SOURCE tag is set to YES (the default) +# and SOURCE_BROWSER tag is set to YES, then the hyperlinks from +# functions in REFERENCES_RELATION and REFERENCED_BY_RELATION lists will +# link to the source code. +# Otherwise they will link to the documentation. + +REFERENCES_LINK_SOURCE = YES + +# If the USE_HTAGS tag is set to YES then the references to source code +# will point to the HTML generated by the htags(1) tool instead of doxygen +# built-in source browser. The htags tool is part of GNU's global source +# tagging system (see http://www.gnu.org/software/global/global.html). You +# will need version 4.8.6 or higher. + +USE_HTAGS = NO + +# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen +# will generate a verbatim copy of the header file for each class for +# which an include is specified. Set to NO to disable this. + +VERBATIM_HEADERS = YES + +#--------------------------------------------------------------------------- +# configuration options related to the alphabetical class index +#--------------------------------------------------------------------------- + +# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index +# of all compounds will be generated. Enable this if the project +# contains a lot of classes, structs, unions or interfaces. + +ALPHABETICAL_INDEX = YES + +# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then +# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns +# in which this list will be split (can be a number in the range [1..20]) + +COLS_IN_ALPHA_INDEX = 5 + +# In case all classes in a project start with a common prefix, all +# classes will be put under the same header in the alphabetical index. +# The IGNORE_PREFIX tag can be used to specify one or more prefixes that +# should be ignored while generating the index headers. + +IGNORE_PREFIX = + +#--------------------------------------------------------------------------- +# configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES (the default) Doxygen will +# generate HTML output. + +GENERATE_HTML = YES + +# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `html' will be used as the default path. + +HTML_OUTPUT = html + +# The HTML_FILE_EXTENSION tag can be used to specify the file extension for +# each generated HTML page (for example: .htm,.php,.asp). If it is left blank +# doxygen will generate files with .html extension. + +HTML_FILE_EXTENSION = .html + +# The HTML_HEADER tag can be used to specify a personal HTML header for +# each generated HTML page. If it is left blank doxygen will generate a +# standard header. Note that when using a custom header you are responsible +# for the proper inclusion of any scripts and style sheets that doxygen +# needs, which is dependent on the configuration options used. +# It is adviced to generate a default header using "doxygen -w html +# header.html footer.html stylesheet.css YourConfigFile" and then modify +# that header. Note that the header is subject to change so you typically +# have to redo this when upgrading to a newer version of doxygen or when changing the value of configuration settings such as GENERATE_TREEVIEW! + +HTML_HEADER = + +# The HTML_FOOTER tag can be used to specify a personal HTML footer for +# each generated HTML page. If it is left blank doxygen will generate a +# standard footer. + +HTML_FOOTER = + +# The HTML_STYLESHEET tag can be used to specify a user-defined cascading +# style sheet that is used by each HTML page. It can be used to +# fine-tune the look of the HTML output. If the tag is left blank doxygen +# will generate a default style sheet. Note that doxygen will try to copy +# the style sheet file to the HTML output directory, so don't put your own +# stylesheet in the HTML output directory as well, or it will be erased! + +HTML_STYLESHEET = + +# The HTML_EXTRA_FILES tag can be used to specify one or more extra images or +# other source files which should be copied to the HTML output directory. Note +# that these files will be copied to the base HTML output directory. Use the +# $relpath$ marker in the HTML_HEADER and/or HTML_FOOTER files to load these +# files. In the HTML_STYLESHEET file, use the file name only. Also note that +# the files will be copied as-is; there are no commands or markers available. + +HTML_EXTRA_FILES = + +# The HTML_COLORSTYLE_HUE tag controls the color of the HTML output. +# Doxygen will adjust the colors in the stylesheet and background images +# according to this color. Hue is specified as an angle on a colorwheel, +# see http://en.wikipedia.org/wiki/Hue for more information. +# For instance the value 0 represents red, 60 is yellow, 120 is green, +# 180 is cyan, 240 is blue, 300 purple, and 360 is red again. +# The allowed range is 0 to 359. + +HTML_COLORSTYLE_HUE = 220 + +# The HTML_COLORSTYLE_SAT tag controls the purity (or saturation) of +# the colors in the HTML output. For a value of 0 the output will use +# grayscales only. A value of 255 will produce the most vivid colors. + +HTML_COLORSTYLE_SAT = 100 + +# The HTML_COLORSTYLE_GAMMA tag controls the gamma correction applied to +# the luminance component of the colors in the HTML output. Values below +# 100 gradually make the output lighter, whereas values above 100 make +# the output darker. The value divided by 100 is the actual gamma applied, +# so 80 represents a gamma of 0.8, The value 220 represents a gamma of 2.2, +# and 100 does not change the gamma. + +HTML_COLORSTYLE_GAMMA = 80 + +# If the HTML_TIMESTAMP tag is set to YES then the footer of each generated HTML +# page will contain the date and time when the page was generated. Setting +# this to NO can help when comparing the output of multiple runs. + +HTML_TIMESTAMP = YES + +# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, +# files or namespaces will be aligned in HTML using tables. If set to +# NO a bullet list will be used. + +HTML_ALIGN_MEMBERS = YES + +# If the HTML_DYNAMIC_SECTIONS tag is set to YES then the generated HTML +# documentation will contain sections that can be hidden and shown after the +# page has loaded. For this to work a browser that supports +# JavaScript and DHTML is required (for instance Mozilla 1.0+, Firefox +# Netscape 6.0+, Internet explorer 5.0+, Konqueror, or Safari). + +HTML_DYNAMIC_SECTIONS = NO + +# If the GENERATE_DOCSET tag is set to YES, additional index files +# will be generated that can be used as input for Apple's Xcode 3 +# integrated development environment, introduced with OSX 10.5 (Leopard). +# To create a documentation set, doxygen will generate a Makefile in the +# HTML output directory. Running make will produce the docset in that +# directory and running "make install" will install the docset in +# ~/Library/Developer/Shared/Documentation/DocSets so that Xcode will find +# it at startup. +# See http://developer.apple.com/tools/creatingdocsetswithdoxygen.html +# for more information. + +GENERATE_DOCSET = NO + +# When GENERATE_DOCSET tag is set to YES, this tag determines the name of the +# feed. A documentation feed provides an umbrella under which multiple +# documentation sets from a single provider (such as a company or product suite) +# can be grouped. + +DOCSET_FEEDNAME = "Doxygen generated docs" + +# When GENERATE_DOCSET tag is set to YES, this tag specifies a string that +# should uniquely identify the documentation set bundle. This should be a +# reverse domain-name style string, e.g. com.mycompany.MyDocSet. Doxygen +# will append .docset to the name. + +DOCSET_BUNDLE_ID = org.doxygen.Project + +# When GENERATE_PUBLISHER_ID tag specifies a string that should uniquely identify +# the documentation publisher. This should be a reverse domain-name style +# string, e.g. com.mycompany.MyDocSet.documentation. + +DOCSET_PUBLISHER_ID = org.doxygen.Publisher + +# The GENERATE_PUBLISHER_NAME tag identifies the documentation publisher. + +DOCSET_PUBLISHER_NAME = Publisher + +# If the GENERATE_HTMLHELP tag is set to YES, additional index files +# will be generated that can be used as input for tools like the +# Microsoft HTML help workshop to generate a compiled HTML help file (.chm) +# of the generated HTML documentation. + +GENERATE_HTMLHELP = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can +# be used to specify the file name of the resulting .chm file. You +# can add a path in front of the file if the result should not be +# written to the html output directory. + +CHM_FILE = + +# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can +# be used to specify the location (absolute path including file name) of +# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run +# the HTML help compiler on the generated index.hhp. + +HHC_LOCATION = + +# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag +# controls if a separate .chi index file is generated (YES) or that +# it should be included in the master .chm file (NO). + +GENERATE_CHI = NO + +# If the GENERATE_HTMLHELP tag is set to YES, the CHM_INDEX_ENCODING +# is used to encode HtmlHelp index (hhk), content (hhc) and project file +# content. + +CHM_INDEX_ENCODING = + +# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag +# controls whether a binary table of contents is generated (YES) or a +# normal table of contents (NO) in the .chm file. + +BINARY_TOC = NO + +# The TOC_EXPAND flag can be set to YES to add extra items for group members +# to the contents of the HTML help documentation and to the tree view. + +TOC_EXPAND = NO + +# If the GENERATE_QHP tag is set to YES and both QHP_NAMESPACE and +# QHP_VIRTUAL_FOLDER are set, an additional index file will be generated +# that can be used as input for Qt's qhelpgenerator to generate a +# Qt Compressed Help (.qch) of the generated HTML documentation. + +GENERATE_QHP = NO + +# If the QHG_LOCATION tag is specified, the QCH_FILE tag can +# be used to specify the file name of the resulting .qch file. +# The path specified is relative to the HTML output folder. + +QCH_FILE = + +# The QHP_NAMESPACE tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#namespace + +QHP_NAMESPACE = org.doxygen.Project + +# The QHP_VIRTUAL_FOLDER tag specifies the namespace to use when generating +# Qt Help Project output. For more information please see +# http://doc.trolltech.com/qthelpproject.html#virtual-folders + +QHP_VIRTUAL_FOLDER = doc + +# If QHP_CUST_FILTER_NAME is set, it specifies the name of a custom filter to +# add. For more information please see +# http://doc.trolltech.com/qthelpproject.html#custom-filters + +QHP_CUST_FILTER_NAME = + +# The QHP_CUST_FILT_ATTRS tag specifies the list of the attributes of the +# custom filter to add. For more information please see +# <a href="http://doc.trolltech.com/qthelpproject.html#custom-filters"> +# Qt Help Project / Custom Filters</a>. + +QHP_CUST_FILTER_ATTRS = + +# The QHP_SECT_FILTER_ATTRS tag specifies the list of the attributes this +# project's +# filter section matches. +# <a href="http://doc.trolltech.com/qthelpproject.html#filter-attributes"> +# Qt Help Project / Filter Attributes</a>. + +QHP_SECT_FILTER_ATTRS = + +# If the GENERATE_QHP tag is set to YES, the QHG_LOCATION tag can +# be used to specify the location of Qt's qhelpgenerator. +# If non-empty doxygen will try to run qhelpgenerator on the generated +# .qhp file. + +QHG_LOCATION = + +# If the GENERATE_ECLIPSEHELP tag is set to YES, additional index files +# will be generated, which together with the HTML files, form an Eclipse help +# plugin. To install this plugin and make it available under the help contents +# menu in Eclipse, the contents of the directory containing the HTML and XML +# files needs to be copied into the plugins directory of eclipse. The name of +# the directory within the plugins directory should be the same as +# the ECLIPSE_DOC_ID value. After copying Eclipse needs to be restarted before +# the help appears. + +GENERATE_ECLIPSEHELP = NO + +# A unique identifier for the eclipse help plugin. When installing the plugin +# the directory name containing the HTML and XML files should also have +# this name. + +ECLIPSE_DOC_ID = org.doxygen.Project + +# The DISABLE_INDEX tag can be used to turn on/off the condensed index at +# top of each HTML page. The value NO (the default) enables the index and +# the value YES disables it. + +DISABLE_INDEX = NO + +# The ENUM_VALUES_PER_LINE tag can be used to set the number of enum values +# (range [0,1..20]) that doxygen will group on one line in the generated HTML +# documentation. Note that a value of 0 will completely suppress the enum +# values from appearing in the overview section. + +ENUM_VALUES_PER_LINE = 4 + +# The GENERATE_TREEVIEW tag is used to specify whether a tree-like index +# structure should be generated to display hierarchical information. +# If the tag value is set to YES, a side panel will be generated +# containing a tree-like index structure (just like the one that +# is generated for HTML Help). For this to work a browser that supports +# JavaScript, DHTML, CSS and frames is required (i.e. any modern browser). +# Windows users are probably better off using the HTML help feature. + +GENERATE_TREEVIEW = NO + +# By enabling USE_INLINE_TREES, doxygen will generate the Groups, Directories, +# and Class Hierarchy pages using a tree view instead of an ordered list. + +USE_INLINE_TREES = NO + +# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be +# used to set the initial width (in pixels) of the frame in which the tree +# is shown. + +TREEVIEW_WIDTH = 250 + +# When the EXT_LINKS_IN_WINDOW option is set to YES doxygen will open +# links to external symbols imported via tag files in a separate window. + +EXT_LINKS_IN_WINDOW = NO + +# Use this tag to change the font size of Latex formulas included +# as images in the HTML documentation. The default is 10. Note that +# when you change the font size after a successful doxygen run you need +# to manually remove any form_*.png images from the HTML output directory +# to force them to be regenerated. + +FORMULA_FONTSIZE = 10 + +# Use the FORMULA_TRANPARENT tag to determine whether or not the images +# generated for formulas are transparent PNGs. Transparent PNGs are +# not supported properly for IE 6.0, but are supported on all modern browsers. +# Note that when changing this option you need to delete any form_*.png files +# in the HTML output before the changes have effect. + +FORMULA_TRANSPARENT = YES + +# Enable the USE_MATHJAX option to render LaTeX formulas using MathJax +# (see http://www.mathjax.org) which uses client side Javascript for the +# rendering instead of using prerendered bitmaps. Use this if you do not +# have LaTeX installed or if you want to formulas look prettier in the HTML +# output. When enabled you also need to install MathJax separately and +# configure the path to it using the MATHJAX_RELPATH option. + +USE_MATHJAX = NO + +# When MathJax is enabled you need to specify the location relative to the +# HTML output directory using the MATHJAX_RELPATH option. The destination +# directory should contain the MathJax.js script. For instance, if the mathjax +# directory is located at the same level as the HTML output directory, then +# MATHJAX_RELPATH should be ../mathjax. The default value points to the +# mathjax.org site, so you can quickly see the result without installing +# MathJax, but it is strongly recommended to install a local copy of MathJax +# before deployment. + +MATHJAX_RELPATH = http://www.mathjax.org/mathjax + +# When the SEARCHENGINE tag is enabled doxygen will generate a search box +# for the HTML output. The underlying search engine uses javascript +# and DHTML and should work on any modern browser. Note that when using +# HTML help (GENERATE_HTMLHELP), Qt help (GENERATE_QHP), or docsets +# (GENERATE_DOCSET) there is already a search function so this one should +# typically be disabled. For large projects the javascript based search engine +# can be slow, then enabling SERVER_BASED_SEARCH may provide a better solution. + +SEARCHENGINE = YES + +# When the SERVER_BASED_SEARCH tag is enabled the search engine will be +# implemented using a PHP enabled web server instead of at the web client +# using Javascript. Doxygen will generate the search PHP script and index +# file to put on the web server. The advantage of the server +# based approach is that it scales better to large projects and allows +# full text search. The disadvantages are that it is more difficult to setup +# and does not have live searching capabilities. + +SERVER_BASED_SEARCH = NO + +#--------------------------------------------------------------------------- +# configuration options related to the LaTeX output +#--------------------------------------------------------------------------- + +# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will +# generate Latex output. + +GENERATE_LATEX = YES + +# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `latex' will be used as the default path. + +LATEX_OUTPUT = latex + +# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be +# invoked. If left blank `latex' will be used as the default command name. +# Note that when enabling USE_PDFLATEX this option is only used for +# generating bitmaps for formulas in the HTML output, but not in the +# Makefile that is written to the output directory. + +LATEX_CMD_NAME = latex + +# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to +# generate index for LaTeX. If left blank `makeindex' will be used as the +# default command name. + +MAKEINDEX_CMD_NAME = makeindex + +# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact +# LaTeX documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_LATEX = NO + +# The PAPER_TYPE tag can be used to set the paper type that is used +# by the printer. Possible values are: a4, letter, legal and +# executive. If left blank a4wide will be used. + +PAPER_TYPE = a4 + +# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX +# packages that should be included in the LaTeX output. + +EXTRA_PACKAGES = + +# The LATEX_HEADER tag can be used to specify a personal LaTeX header for +# the generated latex document. The header should contain everything until +# the first chapter. If it is left blank doxygen will generate a +# standard header. Notice: only use this tag if you know what you are doing! + +LATEX_HEADER = + +# The LATEX_FOOTER tag can be used to specify a personal LaTeX footer for +# the generated latex document. The footer should contain everything after +# the last chapter. If it is left blank doxygen will generate a +# standard footer. Notice: only use this tag if you know what you are doing! + +LATEX_FOOTER = + +# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated +# is prepared for conversion to pdf (using ps2pdf). The pdf file will +# contain links (just like the HTML output) instead of page references +# This makes the output suitable for online browsing using a pdf viewer. + +PDF_HYPERLINKS = YES + +# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of +# plain latex in the generated Makefile. Set this option to YES to get a +# higher quality PDF documentation. + +USE_PDFLATEX = YES + +# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. +# command to the generated LaTeX files. This will instruct LaTeX to keep +# running if errors occur, instead of asking the user for help. +# This option is also used when generating formulas in HTML. + +LATEX_BATCHMODE = NO + +# If LATEX_HIDE_INDICES is set to YES then doxygen will not +# include the index chapters (such as File Index, Compound Index, etc.) +# in the output. + +LATEX_HIDE_INDICES = NO + +# If LATEX_SOURCE_CODE is set to YES then doxygen will include +# source code with syntax highlighting in the LaTeX output. +# Note that which sources are shown also depends on other settings +# such as SOURCE_BROWSER. + +LATEX_SOURCE_CODE = NO + +#--------------------------------------------------------------------------- +# configuration options related to the RTF output +#--------------------------------------------------------------------------- + +# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output +# The RTF output is optimized for Word 97 and may not look very pretty with +# other RTF readers or editors. + +GENERATE_RTF = NO + +# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `rtf' will be used as the default path. + +RTF_OUTPUT = rtf + +# If the COMPACT_RTF tag is set to YES Doxygen generates more compact +# RTF documents. This may be useful for small projects and may help to +# save some trees in general. + +COMPACT_RTF = NO + +# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated +# will contain hyperlink fields. The RTF file will +# contain links (just like the HTML output) instead of page references. +# This makes the output suitable for online browsing using WORD or other +# programs which support those fields. +# Note: wordpad (write) and others do not support links. + +RTF_HYPERLINKS = NO + +# Load stylesheet definitions from file. Syntax is similar to doxygen's +# config file, i.e. a series of assignments. You only have to provide +# replacements, missing definitions are set to their default value. + +RTF_STYLESHEET_FILE = + +# Set optional variables used in the generation of an rtf document. +# Syntax is similar to doxygen's config file. + +RTF_EXTENSIONS_FILE = + +#--------------------------------------------------------------------------- +# configuration options related to the man page output +#--------------------------------------------------------------------------- + +# If the GENERATE_MAN tag is set to YES (the default) Doxygen will +# generate man pages + +GENERATE_MAN = NO + +# The MAN_OUTPUT tag is used to specify where the man pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `man' will be used as the default path. + +MAN_OUTPUT = man + +# The MAN_EXTENSION tag determines the extension that is added to +# the generated man pages (default is the subroutine's section .3) + +MAN_EXTENSION = .3 + +# If the MAN_LINKS tag is set to YES and Doxygen generates man output, +# then it will generate one additional man file for each entity +# documented in the real man page(s). These additional files +# only source the real man page, but without them the man command +# would be unable to find the correct page. The default is NO. + +MAN_LINKS = NO + +#--------------------------------------------------------------------------- +# configuration options related to the XML output +#--------------------------------------------------------------------------- + +# If the GENERATE_XML tag is set to YES Doxygen will +# generate an XML file that captures the structure of +# the code including all documentation. + +GENERATE_XML = NO + +# The XML_OUTPUT tag is used to specify where the XML pages will be put. +# If a relative path is entered the value of OUTPUT_DIRECTORY will be +# put in front of it. If left blank `xml' will be used as the default path. + +XML_OUTPUT = xml + +# The XML_SCHEMA tag can be used to specify an XML schema, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_SCHEMA = + +# The XML_DTD tag can be used to specify an XML DTD, +# which can be used by a validating XML parser to check the +# syntax of the XML files. + +XML_DTD = + +# If the XML_PROGRAMLISTING tag is set to YES Doxygen will +# dump the program listings (including syntax highlighting +# and cross-referencing information) to the XML output. Note that +# enabling this will significantly increase the size of the XML output. + +XML_PROGRAMLISTING = YES + +#--------------------------------------------------------------------------- +# configuration options for the AutoGen Definitions output +#--------------------------------------------------------------------------- + +# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will +# generate an AutoGen Definitions (see autogen.sf.net) file +# that captures the structure of the code including all +# documentation. Note that this feature is still experimental +# and incomplete at the moment. + +GENERATE_AUTOGEN_DEF = NO + +#--------------------------------------------------------------------------- +# configuration options related to the Perl module output +#--------------------------------------------------------------------------- + +# If the GENERATE_PERLMOD tag is set to YES Doxygen will +# generate a Perl module file that captures the structure of +# the code including all documentation. Note that this +# feature is still experimental and incomplete at the +# moment. + +GENERATE_PERLMOD = NO + +# If the PERLMOD_LATEX tag is set to YES Doxygen will generate +# the necessary Makefile rules, Perl scripts and LaTeX code to be able +# to generate PDF and DVI output from the Perl module output. + +PERLMOD_LATEX = NO + +# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be +# nicely formatted so it can be parsed by a human reader. +# This is useful +# if you want to understand what is going on. +# On the other hand, if this +# tag is set to NO the size of the Perl module output will be much smaller +# and Perl will parse it just the same. + +PERLMOD_PRETTY = YES + +# The names of the make variables in the generated doxyrules.make file +# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. +# This is useful so different doxyrules.make files included by the same +# Makefile don't overwrite each other's variables. + +PERLMOD_MAKEVAR_PREFIX = + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will +# evaluate all C-preprocessor directives found in the sources and include +# files. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro +# names in the source code. If set to NO (the default) only conditional +# compilation will be performed. Macro expansion can be done in a controlled +# way by setting EXPAND_ONLY_PREDEF to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES +# then the macro expansion is limited to the macros specified with the +# PREDEFINED and EXPAND_AS_DEFINED tags. + +EXPAND_ONLY_PREDEF = NO + +# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files +# pointed to by INCLUDE_PATH will be searched when a #include is found. + +SEARCH_INCLUDES = YES + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by +# the preprocessor. + +INCLUDE_PATH = + +# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard +# patterns (like *.h and *.hpp) to filter out the header-files in the +# directories. If left blank, the patterns specified with FILE_PATTERNS will +# be used. + +INCLUDE_FILE_PATTERNS = + +# The PREDEFINED tag can be used to specify one or more macro names that +# are defined before the preprocessor is started (similar to the -D option of +# gcc). The argument of the tag is a list of macros of the form: name +# or name=definition (no spaces). If the definition and the = are +# omitted =1 is assumed. To prevent a macro definition from being +# undefined via #undef or recursively expanded use the := operator +# instead of the = operator. + +PREDEFINED = + +# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then +# this tag can be used to specify a list of macro names that should be expanded. +# The macro definition that is found in the sources will be used. +# Use the PREDEFINED tag if you want to use a different macro definition that +# overrules the definition found in the source code. + +EXPAND_AS_DEFINED = + +# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then +# doxygen's preprocessor will remove all references to function-like macros +# that are alone on a line, have an all uppercase name, and do not end with a +# semicolon, because these will confuse the parser if not removed. + +SKIP_FUNCTION_MACROS = YES + +#--------------------------------------------------------------------------- +# Configuration::additions related to external references +#--------------------------------------------------------------------------- + +# The TAGFILES option can be used to specify one or more tagfiles. +# Optionally an initial location of the external documentation +# can be added for each tagfile. The format of a tag file without +# this location is as follows: +# +# TAGFILES = file1 file2 ... +# Adding location for the tag files is done as follows: +# +# TAGFILES = file1=loc1 "file2 = loc2" ... +# where "loc1" and "loc2" can be relative or absolute paths or +# URLs. If a location is present for each tag, the installdox tool +# does not have to be run to correct the links. +# Note that each tag file must have a unique name +# (where the name does NOT include the path) +# If a tag file is not located in the directory in which doxygen +# is run, you must also specify the path to the tagfile here. + +TAGFILES = + +# When a file name is specified after GENERATE_TAGFILE, doxygen will create +# a tag file that is based on the input files it reads. + +GENERATE_TAGFILE = + +# If the ALLEXTERNALS tag is set to YES all external classes will be listed +# in the class index. If set to NO only the inherited external classes +# will be listed. + +ALLEXTERNALS = NO + +# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed +# in the modules index. If set to NO, only the current project's groups will +# be listed. + +EXTERNAL_GROUPS = YES + +# The PERL_PATH should be the absolute path and name of the perl script +# interpreter (i.e. the result of `which perl'). + +PERL_PATH = /usr/bin/perl + +#--------------------------------------------------------------------------- +# Configuration options related to the dot tool +#--------------------------------------------------------------------------- + +# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will +# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base +# or super classes. Setting the tag to NO turns the diagrams off. Note that +# this option also works with HAVE_DOT disabled, but it is recommended to +# install and use dot, since it yields more powerful graphs. + +CLASS_DIAGRAMS = YES + +# You can define message sequence charts within doxygen comments using the \msc +# command. Doxygen will then run the mscgen tool (see +# http://www.mcternan.me.uk/mscgen/) to produce the chart and insert it in the +# documentation. The MSCGEN_PATH tag allows you to specify the directory where +# the mscgen tool resides. If left empty the tool is assumed to be found in the +# default search path. + +MSCGEN_PATH = + +# If set to YES, the inheritance and collaboration graphs will hide +# inheritance and usage relations if the target is undocumented +# or is not a class. + +HIDE_UNDOC_RELATIONS = YES + +# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is +# available from the path. This tool is part of Graphviz, a graph visualization +# toolkit from AT&T and Lucent Bell Labs. The other options in this section +# have no effect if this option is set to NO (the default) + +HAVE_DOT = NO + +# The DOT_NUM_THREADS specifies the number of dot invocations doxygen is +# allowed to run in parallel. When set to 0 (the default) doxygen will +# base this on the number of processors available in the system. You can set it +# explicitly to a value larger than 0 to get control over the balance +# between CPU load and processing speed. + +DOT_NUM_THREADS = 0 + +# By default doxygen will write a font called Helvetica to the output +# directory and reference it in all dot files that doxygen generates. +# When you want a differently looking font you can specify the font name +# using DOT_FONTNAME. You need to make sure dot is able to find the font, +# which can be done by putting it in a standard location or by setting the +# DOTFONTPATH environment variable or by setting DOT_FONTPATH to the directory +# containing the font. + +DOT_FONTNAME = Helvetica + +# The DOT_FONTSIZE tag can be used to set the size of the font of dot graphs. +# The default size is 10pt. + +DOT_FONTSIZE = 10 + +# By default doxygen will tell dot to use the output directory to look for the +# FreeSans.ttf font (which doxygen will put there itself). If you specify a +# different font using DOT_FONTNAME you can set the path where dot +# can find it using this tag. + +DOT_FONTPATH = + +# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect inheritance relations. Setting this tag to YES will force the +# the CLASS_DIAGRAMS tag to NO. + +CLASS_GRAPH = YES + +# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for each documented class showing the direct and +# indirect implementation dependencies (inheritance, containment, and +# class references variables) of the class with other documented classes. + +COLLABORATION_GRAPH = YES + +# If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen +# will generate a graph for groups, showing the direct groups dependencies + +GROUP_GRAPHS = YES + +# If the UML_LOOK tag is set to YES doxygen will generate inheritance and +# collaboration diagrams in a style similar to the OMG's Unified Modeling +# Language. + +UML_LOOK = NO + +# If set to YES, the inheritance and collaboration graphs will show the +# relations between templates and their instances. + +TEMPLATE_RELATIONS = NO + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT +# tags are set to YES then doxygen will generate a graph for each documented +# file showing the direct and indirect include dependencies of the file with +# other documented files. + +INCLUDE_GRAPH = YES + +# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and +# HAVE_DOT tags are set to YES then doxygen will generate a graph for each +# documented header file showing the documented files that directly or +# indirectly include this file. + +INCLUDED_BY_GRAPH = YES + +# If the CALL_GRAPH and HAVE_DOT options are set to YES then +# doxygen will generate a call dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable call graphs +# for selected functions only using the \callgraph command. + +CALL_GRAPH = NO + +# If the CALLER_GRAPH and HAVE_DOT tags are set to YES then +# doxygen will generate a caller dependency graph for every global function +# or class method. Note that enabling this option will significantly increase +# the time of a run. So in most cases it will be better to enable caller +# graphs for selected functions only using the \callergraph command. + +CALLER_GRAPH = NO + +# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen +# will generate a graphical hierarchy of all classes instead of a textual one. + +GRAPHICAL_HIERARCHY = YES + +# If the DIRECTORY_GRAPH, SHOW_DIRECTORIES and HAVE_DOT tags are set to YES +# then doxygen will show the dependencies a directory has on other directories +# in a graphical way. The dependency relations are determined by the #include +# relations between the files in the directories. + +DIRECTORY_GRAPH = YES + +# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images +# generated by dot. Possible values are svg, png, jpg, or gif. +# If left blank png will be used. + +DOT_IMAGE_FORMAT = png + +# The tag DOT_PATH can be used to specify the path where the dot tool can be +# found. If left blank, it is assumed the dot tool can be found in the path. + +DOT_PATH = + +# The DOTFILE_DIRS tag can be used to specify one or more directories that +# contain dot files that are included in the documentation (see the +# \dotfile command). + +DOTFILE_DIRS = + +# The MSCFILE_DIRS tag can be used to specify one or more directories that +# contain msc files that are included in the documentation (see the +# \mscfile command). + +MSCFILE_DIRS = + +# The DOT_GRAPH_MAX_NODES tag can be used to set the maximum number of +# nodes that will be shown in the graph. If the number of nodes in a graph +# becomes larger than this value, doxygen will truncate the graph, which is +# visualized by representing a node as a red box. Note that doxygen if the +# number of direct children of the root node in a graph is already larger than +# DOT_GRAPH_MAX_NODES then the graph will not be shown at all. Also note +# that the size of a graph can be further restricted by MAX_DOT_GRAPH_DEPTH. + +DOT_GRAPH_MAX_NODES = 50 + +# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the +# graphs generated by dot. A depth value of 3 means that only nodes reachable +# from the root by following a path via at most 3 edges will be shown. Nodes +# that lay further from the root node will be omitted. Note that setting this +# option to 1 or 2 may greatly reduce the computation time needed for large +# code bases. Also note that the size of a graph can be further restricted by +# DOT_GRAPH_MAX_NODES. Using a depth of 0 means no depth restriction. + +MAX_DOT_GRAPH_DEPTH = 0 + +# Set the DOT_TRANSPARENT tag to YES to generate images with a transparent +# background. This is disabled by default, because dot on Windows does not +# seem to support this out of the box. Warning: Depending on the platform used, +# enabling this option may lead to badly anti-aliased labels on the edges of +# a graph (i.e. they become hard to read). + +DOT_TRANSPARENT = NO + +# Set the DOT_MULTI_TARGETS tag to YES allow dot to generate multiple output +# files in one run (i.e. multiple -o and -T options on the command line). This +# makes dot run faster, but since only newer versions of dot (>1.8.10) +# support this, this feature is disabled by default. + +DOT_MULTI_TARGETS = NO + +# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will +# generate a legend page explaining the meaning of the various boxes and +# arrows in the dot generated graphs. + +GENERATE_LEGEND = YES + +# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will +# remove the intermediate dot files that are used to generate +# the various graphs. + +DOT_CLEANUP = YES diff --git a/src/gallium/frontends/clover/Makefile.sources b/src/gallium/frontends/clover/Makefile.sources new file mode 100644 index 00000000000..38f94981fb6 --- /dev/null +++ b/src/gallium/frontends/clover/Makefile.sources @@ -0,0 +1,68 @@ +CPP_SOURCES := \ + api/context.cpp \ + api/device.cpp \ + api/dispatch.cpp \ + api/dispatch.hpp \ + api/event.cpp \ + api/interop.cpp \ + api/kernel.cpp \ + api/memory.cpp \ + api/platform.cpp \ + api/program.cpp \ + api/queue.cpp \ + api/sampler.cpp \ + api/transfer.cpp \ + api/util.hpp \ + core/context.cpp \ + core/context.hpp \ + core/device.cpp \ + core/device.hpp \ + core/error.hpp \ + core/event.cpp \ + core/event.hpp \ + core/format.cpp \ + core/format.hpp \ + core/kernel.cpp \ + core/kernel.hpp \ + core/memory.cpp \ + core/memory.hpp \ + core/module.cpp \ + core/module.hpp \ + core/object.hpp \ + core/platform.cpp \ + core/platform.hpp \ + core/program.cpp \ + core/program.hpp \ + core/property.hpp \ + core/queue.cpp \ + core/queue.hpp \ + core/resource.cpp \ + core/resource.hpp \ + core/sampler.cpp \ + core/sampler.hpp \ + core/timestamp.cpp \ + core/timestamp.hpp \ + util/adaptor.hpp \ + util/algebra.hpp \ + util/algorithm.hpp \ + util/factor.hpp \ + util/functional.hpp \ + util/lazy.hpp \ + util/pointer.hpp \ + util/range.hpp \ + util/tuple.hpp + +LLVM_SOURCES := \ + llvm/codegen/bitcode.cpp \ + llvm/codegen/common.cpp \ + llvm/codegen/native.cpp \ + llvm/codegen.hpp \ + llvm/compat.hpp \ + llvm/invocation.cpp \ + llvm/invocation.hpp \ + llvm/metadata.hpp \ + llvm/util.hpp + +SPIRV_SOURCES := \ + spirv/invocation.cpp \ + spirv/invocation.hpp diff --git a/src/gallium/frontends/clover/api/context.cpp b/src/gallium/frontends/clover/api/context.cpp new file mode 100644 index 00000000000..c0cd2d32b95 --- /dev/null +++ b/src/gallium/frontends/clover/api/context.cpp @@ -0,0 +1,144 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "api/util.hpp" +#include "core/context.hpp" +#include "core/platform.hpp" + +using namespace clover; + +CLOVER_API cl_context +clCreateContext(const cl_context_properties *d_props, cl_uint num_devs, + const cl_device_id *d_devs, + void (CL_CALLBACK *pfn_notify)(const char *, const void *, + size_t, void *), + void *user_data, cl_int *r_errcode) try { + auto props = obj<property_list_tag>(d_props); + auto devs = objs(d_devs, num_devs); + + if (!pfn_notify && user_data) + throw error(CL_INVALID_VALUE); + + for (auto &prop : props) { + if (prop.first == CL_CONTEXT_PLATFORM) + obj(prop.second.as<cl_platform_id>()); + else + throw error(CL_INVALID_PROPERTY); + } + + const auto notify = (!pfn_notify ? context::notify_action() : + [=](const char *s) { + pfn_notify(s, NULL, 0, user_data); + }); + + ret_error(r_errcode, CL_SUCCESS); + return desc(new context(props, devs, notify)); + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_context +clCreateContextFromType(const cl_context_properties *d_props, + cl_device_type type, + void (CL_CALLBACK *pfn_notify)( + const char *, const void *, size_t, void *), + void *user_data, cl_int *r_errcode) try { + cl_platform_id d_platform; + cl_uint num_platforms; + cl_int ret; + std::vector<cl_device_id> devs; + cl_uint num_devices; + + ret = clGetPlatformIDs(1, &d_platform, &num_platforms); + if (ret || !num_platforms) + throw error(CL_INVALID_PLATFORM); + + ret = clGetDeviceIDs(d_platform, type, 0, NULL, &num_devices); + if (ret) + throw error(CL_DEVICE_NOT_FOUND); + devs.resize(num_devices); + ret = clGetDeviceIDs(d_platform, type, num_devices, devs.data(), 0); + if (ret) + throw error(CL_DEVICE_NOT_FOUND); + + return clCreateContext(d_props, num_devices, devs.data(), pfn_notify, + user_data, r_errcode); + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_int +clRetainContext(cl_context d_ctx) try { + obj(d_ctx).retain(); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clReleaseContext(cl_context d_ctx) try { + if (obj(d_ctx).release()) + delete pobj(d_ctx); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetContextInfo(cl_context d_ctx, cl_context_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &ctx = obj(d_ctx); + + switch (param) { + case CL_CONTEXT_REFERENCE_COUNT: + buf.as_scalar<cl_uint>() = ctx.ref_count(); + break; + + case CL_CONTEXT_NUM_DEVICES: + buf.as_scalar<cl_uint>() = ctx.devices().size(); + break; + + case CL_CONTEXT_DEVICES: + buf.as_vector<cl_device_id>() = descs(ctx.devices()); + break; + + case CL_CONTEXT_PROPERTIES: + buf.as_vector<cl_context_properties>() = desc(ctx.properties()); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} diff --git a/src/gallium/frontends/clover/api/device.cpp b/src/gallium/frontends/clover/api/device.cpp new file mode 100644 index 00000000000..042f2eda21c --- /dev/null +++ b/src/gallium/frontends/clover/api/device.cpp @@ -0,0 +1,421 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "api/util.hpp" +#include "core/platform.hpp" +#include "core/device.hpp" +#include "git_sha1.h" + +using namespace clover; + +CLOVER_API cl_int +clGetDeviceIDs(cl_platform_id d_platform, cl_device_type device_type, + cl_uint num_entries, cl_device_id *rd_devices, + cl_uint *rnum_devices) try { + auto &platform = obj(d_platform); + std::vector<cl_device_id> d_devs; + + if ((!num_entries && rd_devices) || + (!rnum_devices && !rd_devices)) + throw error(CL_INVALID_VALUE); + + // Collect matching devices + for (device &dev : platform) { + if (((device_type & CL_DEVICE_TYPE_DEFAULT) && + dev == platform.front()) || + (device_type & dev.type())) + d_devs.push_back(desc(dev)); + } + + if (d_devs.empty()) + throw error(CL_DEVICE_NOT_FOUND); + + // ...and return the requested data. + if (rnum_devices) + *rnum_devices = d_devs.size(); + if (rd_devices) + copy(range(d_devs.begin(), + std::min((unsigned)d_devs.size(), num_entries)), + rd_devices); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clCreateSubDevices(cl_device_id d_dev, + const cl_device_partition_property *props, + cl_uint num_devs, cl_device_id *rd_devs, + cl_uint *rnum_devs) { + // There are no currently supported partitioning schemes. + return CL_INVALID_VALUE; +} + +CLOVER_API cl_int +clRetainDevice(cl_device_id d_dev) try { + obj(d_dev); + + // The reference count doesn't change for root devices. + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clReleaseDevice(cl_device_id d_dev) try { + obj(d_dev); + + // The reference count doesn't change for root devices. + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetDeviceInfo(cl_device_id d_dev, cl_device_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &dev = obj(d_dev); + + switch (param) { + case CL_DEVICE_TYPE: + buf.as_scalar<cl_device_type>() = dev.type(); + break; + + case CL_DEVICE_VENDOR_ID: + buf.as_scalar<cl_uint>() = dev.vendor_id(); + break; + + case CL_DEVICE_MAX_COMPUTE_UNITS: + buf.as_scalar<cl_uint>() = dev.max_compute_units(); + break; + + case CL_DEVICE_MAX_WORK_ITEM_DIMENSIONS: + buf.as_scalar<cl_uint>() = dev.max_block_size().size(); + break; + + case CL_DEVICE_MAX_WORK_ITEM_SIZES: + buf.as_vector<size_t>() = dev.max_block_size(); + break; + + case CL_DEVICE_MAX_WORK_GROUP_SIZE: + buf.as_scalar<size_t>() = dev.max_threads_per_block(); + break; + + case CL_DEVICE_PREFERRED_VECTOR_WIDTH_CHAR: + buf.as_scalar<cl_uint>() = 16; + break; + + case CL_DEVICE_PREFERRED_VECTOR_WIDTH_SHORT: + buf.as_scalar<cl_uint>() = 8; + break; + + case CL_DEVICE_PREFERRED_VECTOR_WIDTH_INT: + buf.as_scalar<cl_uint>() = 4; + break; + + case CL_DEVICE_PREFERRED_VECTOR_WIDTH_LONG: + buf.as_scalar<cl_uint>() = 2; + break; + + case CL_DEVICE_PREFERRED_VECTOR_WIDTH_FLOAT: + buf.as_scalar<cl_uint>() = 4; + break; + + case CL_DEVICE_PREFERRED_VECTOR_WIDTH_DOUBLE: + buf.as_scalar<cl_uint>() = dev.has_doubles() ? 2 : 0; + break; + + case CL_DEVICE_PREFERRED_VECTOR_WIDTH_HALF: + buf.as_scalar<cl_uint>() = dev.has_halves() ? 8 : 0; + break; + + case CL_DEVICE_MAX_CLOCK_FREQUENCY: + buf.as_scalar<cl_uint>() = dev.max_clock_frequency(); + break; + + case CL_DEVICE_ADDRESS_BITS: + buf.as_scalar<cl_uint>() = dev.address_bits(); + break; + + case CL_DEVICE_MAX_READ_IMAGE_ARGS: + buf.as_scalar<cl_uint>() = dev.max_images_read(); + break; + + case CL_DEVICE_MAX_WRITE_IMAGE_ARGS: + buf.as_scalar<cl_uint>() = dev.max_images_write(); + break; + + case CL_DEVICE_MAX_MEM_ALLOC_SIZE: + buf.as_scalar<cl_ulong>() = dev.max_mem_alloc_size(); + break; + + case CL_DEVICE_IMAGE2D_MAX_WIDTH: + case CL_DEVICE_IMAGE2D_MAX_HEIGHT: + buf.as_scalar<size_t>() = 1 << dev.max_image_levels_2d(); + break; + + case CL_DEVICE_IMAGE3D_MAX_WIDTH: + case CL_DEVICE_IMAGE3D_MAX_HEIGHT: + case CL_DEVICE_IMAGE3D_MAX_DEPTH: + buf.as_scalar<size_t>() = 1 << dev.max_image_levels_3d(); + break; + + case CL_DEVICE_IMAGE_MAX_BUFFER_SIZE: + buf.as_scalar<size_t>() = dev.max_image_buffer_size(); + break; + + case CL_DEVICE_IMAGE_MAX_ARRAY_SIZE: + buf.as_scalar<size_t>() = dev.max_image_array_number(); + break; + + case CL_DEVICE_IMAGE_SUPPORT: + buf.as_scalar<cl_bool>() = dev.image_support(); + break; + + case CL_DEVICE_MAX_PARAMETER_SIZE: + buf.as_scalar<size_t>() = dev.max_mem_input(); + break; + + case CL_DEVICE_MAX_SAMPLERS: + buf.as_scalar<cl_uint>() = dev.max_samplers(); + break; + + case CL_DEVICE_MEM_BASE_ADDR_ALIGN: + buf.as_scalar<cl_uint>() = 8 * + std::max(dev.mem_base_addr_align(), (cl_uint) sizeof(cl_long) * 16); + break; + + case CL_DEVICE_MIN_DATA_TYPE_ALIGN_SIZE: + buf.as_scalar<cl_uint>() = 128; + break; + + case CL_DEVICE_HALF_FP_CONFIG: + // This is the "mandated minimum half precision floating-point + // capability" for OpenCL 1.x. + buf.as_scalar<cl_device_fp_config>() = + CL_FP_INF_NAN | CL_FP_ROUND_TO_NEAREST; + break; + + case CL_DEVICE_SINGLE_FP_CONFIG: + // This is the "mandated minimum single precision floating-point + // capability" for OpenCL 1.1. In OpenCL 1.2, nothing is required for + // custom devices. + buf.as_scalar<cl_device_fp_config>() = + CL_FP_INF_NAN | CL_FP_ROUND_TO_NEAREST; + break; + + case CL_DEVICE_DOUBLE_FP_CONFIG: + if (dev.has_doubles()) + // This is the "mandated minimum double precision floating-point + // capability" + buf.as_scalar<cl_device_fp_config>() = + CL_FP_FMA + | CL_FP_ROUND_TO_NEAREST + | CL_FP_ROUND_TO_ZERO + | CL_FP_ROUND_TO_INF + | CL_FP_INF_NAN + | CL_FP_DENORM; + else + buf.as_scalar<cl_device_fp_config>() = 0; + break; + + case CL_DEVICE_GLOBAL_MEM_CACHE_TYPE: + buf.as_scalar<cl_device_mem_cache_type>() = CL_NONE; + break; + + case CL_DEVICE_GLOBAL_MEM_CACHELINE_SIZE: + buf.as_scalar<cl_uint>() = 0; + break; + + case CL_DEVICE_GLOBAL_MEM_CACHE_SIZE: + buf.as_scalar<cl_ulong>() = 0; + break; + + case CL_DEVICE_GLOBAL_MEM_SIZE: + buf.as_scalar<cl_ulong>() = dev.max_mem_global(); + break; + + case CL_DEVICE_MAX_CONSTANT_BUFFER_SIZE: + buf.as_scalar<cl_ulong>() = dev.max_const_buffer_size(); + break; + + case CL_DEVICE_MAX_CONSTANT_ARGS: + buf.as_scalar<cl_uint>() = dev.max_const_buffers(); + break; + + case CL_DEVICE_LOCAL_MEM_TYPE: + buf.as_scalar<cl_device_local_mem_type>() = CL_LOCAL; + break; + + case CL_DEVICE_LOCAL_MEM_SIZE: + buf.as_scalar<cl_ulong>() = dev.max_mem_local(); + break; + + case CL_DEVICE_ERROR_CORRECTION_SUPPORT: + buf.as_scalar<cl_bool>() = CL_FALSE; + break; + + case CL_DEVICE_PROFILING_TIMER_RESOLUTION: + buf.as_scalar<size_t>() = 0; + break; + + case CL_DEVICE_ENDIAN_LITTLE: + buf.as_scalar<cl_bool>() = (dev.endianness() == PIPE_ENDIAN_LITTLE); + break; + + case CL_DEVICE_AVAILABLE: + case CL_DEVICE_COMPILER_AVAILABLE: + case CL_DEVICE_LINKER_AVAILABLE: + buf.as_scalar<cl_bool>() = CL_TRUE; + break; + + case CL_DEVICE_EXECUTION_CAPABILITIES: + buf.as_scalar<cl_device_exec_capabilities>() = CL_EXEC_KERNEL; + break; + + case CL_DEVICE_QUEUE_PROPERTIES: + buf.as_scalar<cl_command_queue_properties>() = CL_QUEUE_PROFILING_ENABLE; + break; + + case CL_DEVICE_BUILT_IN_KERNELS: + buf.as_string() = ""; + break; + + case CL_DEVICE_NAME: + buf.as_string() = dev.device_name(); + break; + + case CL_DEVICE_VENDOR: + buf.as_string() = dev.vendor_name(); + break; + + case CL_DRIVER_VERSION: + buf.as_string() = PACKAGE_VERSION; + break; + + case CL_DEVICE_PROFILE: + buf.as_string() = "FULL_PROFILE"; + break; + + case CL_DEVICE_VERSION: + buf.as_string() = "OpenCL " + dev.device_version() + " Mesa " PACKAGE_VERSION MESA_GIT_SHA1; + break; + + case CL_DEVICE_EXTENSIONS: + buf.as_string() = dev.supported_extensions(); + break; + + case CL_DEVICE_PLATFORM: + buf.as_scalar<cl_platform_id>() = desc(dev.platform); + break; + + case CL_DEVICE_HOST_UNIFIED_MEMORY: + buf.as_scalar<cl_bool>() = dev.has_unified_memory(); + break; + + case CL_DEVICE_NATIVE_VECTOR_WIDTH_CHAR: + buf.as_scalar<cl_uint>() = 16; + break; + + case CL_DEVICE_NATIVE_VECTOR_WIDTH_SHORT: + buf.as_scalar<cl_uint>() = 8; + break; + + case CL_DEVICE_NATIVE_VECTOR_WIDTH_INT: + buf.as_scalar<cl_uint>() = 4; + break; + + case CL_DEVICE_NATIVE_VECTOR_WIDTH_LONG: + buf.as_scalar<cl_uint>() = 2; + break; + + case CL_DEVICE_NATIVE_VECTOR_WIDTH_FLOAT: + buf.as_scalar<cl_uint>() = 4; + break; + + case CL_DEVICE_NATIVE_VECTOR_WIDTH_DOUBLE: + buf.as_scalar<cl_uint>() = dev.has_doubles() ? 2 : 0; + break; + + case CL_DEVICE_NATIVE_VECTOR_WIDTH_HALF: + buf.as_scalar<cl_uint>() = dev.has_halves() ? 8 : 0; + break; + + case CL_DEVICE_OPENCL_C_VERSION: + buf.as_string() = "OpenCL C " + dev.device_clc_version() + " "; + break; + + case CL_DEVICE_PRINTF_BUFFER_SIZE: + // Per the spec, the minimum value for the FULL profile is 1 MB. + // However, clover is not ready yet to support it + buf.as_scalar<size_t>() = 0 /* 1024 */; + break; + + case CL_DEVICE_PREFERRED_INTEROP_USER_SYNC: + buf.as_scalar<cl_bool>() = CL_TRUE; + break; + + case CL_DEVICE_PARENT_DEVICE: + buf.as_scalar<cl_device_id>() = NULL; + break; + + case CL_DEVICE_PARTITION_MAX_SUB_DEVICES: + buf.as_scalar<cl_uint>() = 0; + break; + + case CL_DEVICE_PARTITION_PROPERTIES: + buf.as_vector<cl_device_partition_property>() = + desc(property_list<cl_device_partition_property>()); + break; + + case CL_DEVICE_PARTITION_AFFINITY_DOMAIN: + buf.as_scalar<cl_device_affinity_domain>() = 0; + break; + + case CL_DEVICE_PARTITION_TYPE: + buf.as_vector<cl_device_partition_property>() = + desc(property_list<cl_device_partition_property>()); + break; + + case CL_DEVICE_REFERENCE_COUNT: + buf.as_scalar<cl_uint>() = 1; + break; + + case CL_DEVICE_SVM_CAPABILITIES: + case CL_DEVICE_SVM_CAPABILITIES_ARM: + buf.as_scalar<cl_device_svm_capabilities>() = dev.svm_support(); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} diff --git a/src/gallium/frontends/clover/api/dispatch.cpp b/src/gallium/frontends/clover/api/dispatch.cpp new file mode 100644 index 00000000000..6e1b0351afa --- /dev/null +++ b/src/gallium/frontends/clover/api/dispatch.cpp @@ -0,0 +1,174 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "api/dispatch.hpp" + +namespace clover { + const cl_icd_dispatch _dispatch = { + clGetPlatformIDs, + GetPlatformInfo, + clGetDeviceIDs, + clGetDeviceInfo, + clCreateContext, + clCreateContextFromType, + clRetainContext, + clReleaseContext, + clGetContextInfo, + clCreateCommandQueue, + clRetainCommandQueue, + clReleaseCommandQueue, + clGetCommandQueueInfo, + NULL, // clSetCommandQueueProperty + clCreateBuffer, + clCreateImage2D, + clCreateImage3D, + clRetainMemObject, + clReleaseMemObject, + clGetSupportedImageFormats, + clGetMemObjectInfo, + clGetImageInfo, + clCreateSampler, + clRetainSampler, + clReleaseSampler, + clGetSamplerInfo, + clCreateProgramWithSource, + clCreateProgramWithBinary, + clRetainProgram, + clReleaseProgram, + clBuildProgram, + clUnloadCompiler, + clGetProgramInfo, + clGetProgramBuildInfo, + clCreateKernel, + clCreateKernelsInProgram, + clRetainKernel, + clReleaseKernel, + clSetKernelArg, + clGetKernelInfo, + clGetKernelWorkGroupInfo, + clWaitForEvents, + clGetEventInfo, + clRetainEvent, + clReleaseEvent, + clGetEventProfilingInfo, + clFlush, + clFinish, + clEnqueueReadBuffer, + clEnqueueWriteBuffer, + clEnqueueCopyBuffer, + clEnqueueReadImage, + clEnqueueWriteImage, + clEnqueueCopyImage, + clEnqueueCopyImageToBuffer, + clEnqueueCopyBufferToImage, + clEnqueueMapBuffer, + clEnqueueMapImage, + clEnqueueUnmapMemObject, + clEnqueueNDRangeKernel, + clEnqueueTask, + clEnqueueNativeKernel, + clEnqueueMarker, + clEnqueueWaitForEvents, + clEnqueueBarrier, + GetExtensionFunctionAddress, + NULL, // clCreateFromGLBuffer + NULL, // clCreateFromGLTexture2D + NULL, // clCreateFromGLTexture3D + NULL, // clCreateFromGLRenderbuffer + NULL, // clGetGLObjectInfo + NULL, // clGetGLTextureInfo + NULL, // clEnqueueAcquireGLObjects + NULL, // clEnqueueReleaseGLObjects + NULL, // clGetGLContextInfoKHR + NULL, // clGetDeviceIDsFromD3D10KHR + NULL, // clCreateFromD3D10BufferKHR + NULL, // clCreateFromD3D10Texture2DKHR + NULL, // clCreateFromD3D10Texture3DKHR + NULL, // clEnqueueAcquireD3D10ObjectsKHR + NULL, // clEnqueueReleaseD3D10ObjectsKHR + clSetEventCallback, + clCreateSubBuffer, + clSetMemObjectDestructorCallback, + clCreateUserEvent, + clSetUserEventStatus, + clEnqueueReadBufferRect, + clEnqueueWriteBufferRect, + clEnqueueCopyBufferRect, + NULL, // clCreateSubDevicesEXT + NULL, // clRetainDeviceEXT + NULL, // clReleaseDeviceEXT + NULL, // clCreateEventFromGLsyncKHR + clCreateSubDevices, + clRetainDevice, + clReleaseDevice, + clCreateImage, + clCreateProgramWithBuiltInKernels, + clCompileProgram, + clLinkProgram, + clUnloadPlatformCompiler, + clGetKernelArgInfo, + clEnqueueFillBuffer, + clEnqueueFillImage, + clEnqueueMigrateMemObjects, + clEnqueueMarkerWithWaitList, + clEnqueueBarrierWithWaitList, + GetExtensionFunctionAddressForPlatform, + NULL, // clCreateFromGLTexture + NULL, // clGetDeviceIDsFromD3D11KHR + NULL, // clCreateFromD3D11BufferKHR + NULL, // clCreateFromD3D11Texture2DKHR + NULL, // clCreateFromD3D11Texture3DKHR + NULL, // clCreateFromDX9MediaSurfaceKHR + NULL, // clEnqueueAcquireD3D11ObjectsKHR + NULL, // clEnqueueReleaseD3D11ObjectsKHR + NULL, // clGetDeviceIDsFromDX9MediaAdapterKHR + NULL, // clEnqueueAcquireDX9MediaSurfacesKHR + NULL, // clEnqueueReleaseDX9MediaSurfacesKHR + NULL, // clCreateFromEGLImageKHR + NULL, // clEnqueueAcquireEGLObjectsKHR + NULL, // clEnqueueReleaseEGLObjectsKHR + NULL, // clCreateEventFromEGLSyncKHR + clCreateCommandQueueWithProperties, + NULL, // clCreatePipe + NULL, // clGetPipeInfo + clSVMAlloc, + clSVMFree, + clEnqueueSVMFree, + clEnqueueSVMMemcpy, + clEnqueueSVMMemFill, + clEnqueueSVMMap, + clEnqueueSVMUnmap, + NULL, // clCreateSamplerWithProperties + clSetKernelArgSVMPointer, + clSetKernelExecInfo, + NULL, // clGetKernelSubGroupInfoKHR + NULL, // clCloneKernel + NULL, // clCreateProgramWithIL + clEnqueueSVMMigrateMem, + NULL, // clGetDeviceAndHostTimer + NULL, // clGetHostTimer + NULL, // clGetKernelSubGroupInfo + NULL, // clSetDefaultDeviceCommandQueue + NULL, // clSetProgramReleaseCallback + NULL, // clSetProgramSpecializationConstant + }; +} diff --git a/src/gallium/frontends/clover/api/dispatch.hpp b/src/gallium/frontends/clover/api/dispatch.hpp new file mode 100644 index 00000000000..ea835ed6da4 --- /dev/null +++ b/src/gallium/frontends/clover/api/dispatch.hpp @@ -0,0 +1,105 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef API_DISPATCH_HPP +#define API_DISPATCH_HPP + +#include "CL/cl.h" +#include "CL/cl_ext.h" +#include "CL/cl_egl.h" +#include "CL/cl_gl.h" +#include "CL/cl_icd.h" + +namespace clover { + extern const cl_icd_dispatch _dispatch; + + cl_int + GetPlatformInfo(cl_platform_id d_platform, cl_platform_info param, + size_t size, void *r_buf, size_t *r_size); + + void * + GetExtensionFunctionAddress(const char *p_name); + + void * + GetExtensionFunctionAddressForPlatform(cl_platform_id d_platform, + const char *p_name); + + cl_int + IcdGetPlatformIDsKHR(cl_uint num_entries, cl_platform_id *rd_platforms, + cl_uint *rnum_platforms); + + cl_int + EnqueueSVMFree(cl_command_queue command_queue, + cl_uint num_svm_pointers, + void *svm_pointers[], + void (CL_CALLBACK *pfn_free_func) ( + cl_command_queue queue, cl_uint num_svm_pointers, + void *svm_pointers[], void *user_data), + void *user_data, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd); + + cl_int + EnqueueSVMMemcpy(cl_command_queue command_queue, + cl_bool blocking_copy, + void *dst_ptr, + const void *src_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd); + + cl_int + EnqueueSVMMap(cl_command_queue command_queue, + cl_bool blocking_map, + cl_map_flags map_flags, + void *svm_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd); + + cl_int + EnqueueSVMMemFill(cl_command_queue command_queue, + void *svm_ptr, + const void *pattern, + size_t pattern_size, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd); + + cl_int + EnqueueSVMUnmap(cl_command_queue command_queue, + void *svm_ptr, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd); +} + +#endif diff --git a/src/gallium/frontends/clover/api/event.cpp b/src/gallium/frontends/clover/api/event.cpp new file mode 100644 index 00000000000..3f89644d0a4 --- /dev/null +++ b/src/gallium/frontends/clover/api/event.cpp @@ -0,0 +1,309 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "api/util.hpp" +#include "core/event.hpp" + +using namespace clover; + +CLOVER_API cl_event +clCreateUserEvent(cl_context d_ctx, cl_int *r_errcode) try { + auto &ctx = obj(d_ctx); + + ret_error(r_errcode, CL_SUCCESS); + return desc(new soft_event(ctx, {}, false)); + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_int +clSetUserEventStatus(cl_event d_ev, cl_int status) try { + auto &sev = obj<soft_event>(d_ev); + + if (status > 0) + return CL_INVALID_VALUE; + + if (sev.status() <= 0) + return CL_INVALID_OPERATION; + + if (status) + sev.abort(status); + else + sev.trigger(); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clWaitForEvents(cl_uint num_evs, const cl_event *d_evs) try { + auto evs = objs(d_evs, num_evs); + + for (auto &ev : evs) { + if (ev.context() != evs.front().context()) + throw error(CL_INVALID_CONTEXT); + + if (ev.status() < 0) + throw error(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST); + } + + // Create a temporary soft event that depends on all the events in + // the wait list + auto sev = create<soft_event>(evs.front().context(), evs, true); + + // ...and wait on it. + sev().wait(); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetEventInfo(cl_event d_ev, cl_event_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &ev = obj(d_ev); + + switch (param) { + case CL_EVENT_COMMAND_QUEUE: + buf.as_scalar<cl_command_queue>() = desc(ev.queue()); + break; + + case CL_EVENT_CONTEXT: + buf.as_scalar<cl_context>() = desc(ev.context()); + break; + + case CL_EVENT_COMMAND_TYPE: + buf.as_scalar<cl_command_type>() = ev.command(); + break; + + case CL_EVENT_COMMAND_EXECUTION_STATUS: + buf.as_scalar<cl_int>() = ev.status(); + break; + + case CL_EVENT_REFERENCE_COUNT: + buf.as_scalar<cl_uint>() = ev.ref_count(); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clSetEventCallback(cl_event d_ev, cl_int type, + void (CL_CALLBACK *pfn_notify)(cl_event, cl_int, void *), + void *user_data) try { + auto &ev = obj(d_ev); + + if (!pfn_notify || + (type != CL_COMPLETE && type != CL_SUBMITTED && type != CL_RUNNING)) + throw error(CL_INVALID_VALUE); + + // Create a temporary soft event that depends on ev, with + // pfn_notify as completion action. + create<soft_event>(ev.context(), ref_vector<event> { ev }, true, + [=, &ev](event &) { + ev.wait(); + pfn_notify(desc(ev), ev.status(), user_data); + }); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clRetainEvent(cl_event d_ev) try { + obj(d_ev).retain(); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clReleaseEvent(cl_event d_ev) try { + if (obj(d_ev).release()) + delete pobj(d_ev); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueMarker(cl_command_queue d_q, cl_event *rd_ev) try { + auto &q = obj(d_q); + + if (!rd_ev) + throw error(CL_INVALID_VALUE); + + *rd_ev = desc(new hard_event(q, CL_COMMAND_MARKER, {})); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueMarkerWithWaitList(cl_command_queue d_q, cl_uint num_deps, + const cl_event *d_deps, cl_event *rd_ev) try { + auto &q = obj(d_q); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + + for (auto &ev : deps) { + if (ev.context() != q.context()) + throw error(CL_INVALID_CONTEXT); + } + + // Create a hard event that depends on the events in the wait list: + // previous commands in the same queue are implicitly serialized + // with respect to it -- hard events always are. + auto hev = create<hard_event>(q, CL_COMMAND_MARKER, deps); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueBarrier(cl_command_queue d_q) try { + obj(d_q); + + // No need to do anything, q preserves data ordering strictly. + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueBarrierWithWaitList(cl_command_queue d_q, cl_uint num_deps, + const cl_event *d_deps, cl_event *rd_ev) try { + auto &q = obj(d_q); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + + for (auto &ev : deps) { + if (ev.context() != q.context()) + throw error(CL_INVALID_CONTEXT); + } + + // Create a hard event that depends on the events in the wait list: + // subsequent commands in the same queue will be implicitly + // serialized with respect to it -- hard events always are. + auto hev = create<hard_event>(q, CL_COMMAND_BARRIER, deps); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueWaitForEvents(cl_command_queue d_q, cl_uint num_evs, + const cl_event *d_evs) try { + // The wait list is mandatory for clEnqueueWaitForEvents(). + objs(d_evs, num_evs); + + return clEnqueueBarrierWithWaitList(d_q, num_evs, d_evs, NULL); + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetEventProfilingInfo(cl_event d_ev, cl_profiling_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + hard_event &hev = dynamic_cast<hard_event &>(obj(d_ev)); + + if (hev.status() != CL_COMPLETE) + throw error(CL_PROFILING_INFO_NOT_AVAILABLE); + + switch (param) { + case CL_PROFILING_COMMAND_QUEUED: + buf.as_scalar<cl_ulong>() = hev.time_queued(); + break; + + case CL_PROFILING_COMMAND_SUBMIT: + buf.as_scalar<cl_ulong>() = hev.time_submit(); + break; + + case CL_PROFILING_COMMAND_START: + buf.as_scalar<cl_ulong>() = hev.time_start(); + break; + + case CL_PROFILING_COMMAND_END: + buf.as_scalar<cl_ulong>() = hev.time_end(); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (std::bad_cast &e) { + return CL_PROFILING_INFO_NOT_AVAILABLE; + +} catch (lazy<cl_ulong>::undefined_error &e) { + return CL_PROFILING_INFO_NOT_AVAILABLE; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clFinish(cl_command_queue d_q) try { + auto &q = obj(d_q); + + // Create a temporary hard event -- it implicitly depends on all + // the previously queued hard events. + auto hev = create<hard_event>(q, 0, ref_vector<event> {}); + + // And wait on it. + hev().wait(); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} diff --git a/src/gallium/frontends/clover/api/interop.cpp b/src/gallium/frontends/clover/api/interop.cpp new file mode 100644 index 00000000000..b96069f5167 --- /dev/null +++ b/src/gallium/frontends/clover/api/interop.cpp @@ -0,0 +1,69 @@ +// +// Copyright 2015 Advanced Micro Devices, Inc. +// All Rights Reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/event.hpp" +#include "api/util.hpp" + +using namespace clover; + +extern "C" { + +PUBLIC bool +opencl_dri_event_add_ref(cl_event event) +{ + /* This should fail if the event hasn't been created by + * clEnqueueReleaseGLObjects or clEnqueueReleaseEGLObjects. + * + * TODO: implement the CL functions + */ + return false; /*return clRetainEvent(event) == CL_SUCCESS;*/ +} + +PUBLIC bool +opencl_dri_event_release(cl_event event) +{ + return clReleaseEvent(event) == CL_SUCCESS; +} + +PUBLIC bool +opencl_dri_event_wait(cl_event event, uint64_t timeout) try { + if (!timeout) { + return obj(event).status() == CL_COMPLETE; + } + + obj(event).wait(); + return true; + +} catch (error &) { + return false; +} + +PUBLIC struct pipe_fence_handle * +opencl_dri_event_get_fence(cl_event event) try { + return obj(event).fence(); + +} catch (error &) { + return NULL; +} + +} diff --git a/src/gallium/frontends/clover/api/kernel.cpp b/src/gallium/frontends/clover/api/kernel.cpp new file mode 100644 index 00000000000..31a87b63868 --- /dev/null +++ b/src/gallium/frontends/clover/api/kernel.cpp @@ -0,0 +1,390 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "api/util.hpp" +#include "core/kernel.hpp" +#include "core/event.hpp" + +using namespace clover; + +CLOVER_API cl_kernel +clCreateKernel(cl_program d_prog, const char *name, cl_int *r_errcode) try { + auto &prog = obj(d_prog); + + if (!name) + throw error(CL_INVALID_VALUE); + + auto &sym = find(name_equals(name), prog.symbols()); + + ret_error(r_errcode, CL_SUCCESS); + return new kernel(prog, name, range(sym.args)); + +} catch (std::out_of_range &e) { + ret_error(r_errcode, CL_INVALID_KERNEL_NAME); + return NULL; + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_int +clCreateKernelsInProgram(cl_program d_prog, cl_uint count, + cl_kernel *rd_kerns, cl_uint *r_count) try { + auto &prog = obj(d_prog); + auto &syms = prog.symbols(); + + if (rd_kerns && count < syms.size()) + throw error(CL_INVALID_VALUE); + + if (rd_kerns) + copy(map([&](const module::symbol &sym) { + return desc(new kernel(prog, + std::string(sym.name.begin(), + sym.name.end()), + range(sym.args))); + }, syms), + rd_kerns); + + if (r_count) + *r_count = syms.size(); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clRetainKernel(cl_kernel d_kern) try { + obj(d_kern).retain(); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clReleaseKernel(cl_kernel d_kern) try { + if (obj(d_kern).release()) + delete pobj(d_kern); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clSetKernelArg(cl_kernel d_kern, cl_uint idx, size_t size, + const void *value) try { + obj(d_kern).args().at(idx).set(size, value); + return CL_SUCCESS; + +} catch (std::out_of_range &e) { + return CL_INVALID_ARG_INDEX; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetKernelInfo(cl_kernel d_kern, cl_kernel_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &kern = obj(d_kern); + + switch (param) { + case CL_KERNEL_FUNCTION_NAME: + buf.as_string() = kern.name(); + break; + + case CL_KERNEL_NUM_ARGS: + buf.as_scalar<cl_uint>() = kern.args().size(); + break; + + case CL_KERNEL_REFERENCE_COUNT: + buf.as_scalar<cl_uint>() = kern.ref_count(); + break; + + case CL_KERNEL_CONTEXT: + buf.as_scalar<cl_context>() = desc(kern.program().context()); + break; + + case CL_KERNEL_PROGRAM: + buf.as_scalar<cl_program>() = desc(kern.program()); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetKernelWorkGroupInfo(cl_kernel d_kern, cl_device_id d_dev, + cl_kernel_work_group_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &kern = obj(d_kern); + auto &dev = (d_dev ? *pobj(d_dev) : unique(kern.program().devices())); + + if (!count(dev, kern.program().devices())) + throw error(CL_INVALID_DEVICE); + + switch (param) { + case CL_KERNEL_WORK_GROUP_SIZE: + buf.as_scalar<size_t>() = dev.max_threads_per_block(); + break; + + case CL_KERNEL_COMPILE_WORK_GROUP_SIZE: + buf.as_vector<size_t>() = kern.required_block_size(); + break; + + case CL_KERNEL_LOCAL_MEM_SIZE: + buf.as_scalar<cl_ulong>() = kern.mem_local(); + break; + + case CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE: + buf.as_scalar<size_t>() = dev.subgroup_size(); + break; + + case CL_KERNEL_PRIVATE_MEM_SIZE: + buf.as_scalar<cl_ulong>() = kern.mem_private(); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); + +} catch (std::out_of_range &e) { + return CL_INVALID_DEVICE; +} + +CLOVER_API cl_int +clGetKernelArgInfo(cl_kernel d_kern, + cl_uint idx, cl_kernel_arg_info param, + size_t size, void *r_buf, size_t *r_size) { + CLOVER_NOT_SUPPORTED_UNTIL("1.2"); + return CL_KERNEL_ARG_INFO_NOT_AVAILABLE; +} + +namespace { + /// + /// Common argument checking shared by kernel invocation commands. + /// + void + validate_common(const command_queue &q, kernel &kern, + const ref_vector<event> &deps) { + if (kern.program().context() != q.context() || + any_of([&](const event &ev) { + return ev.context() != q.context(); + }, deps)) + throw error(CL_INVALID_CONTEXT); + + if (any_of([](kernel::argument &arg) { + return !arg.set(); + }, kern.args())) + throw error(CL_INVALID_KERNEL_ARGS); + + // If the command queue's device is not associated to the program, we get + // a module, with no sections, which will also fail the following test. + auto &m = kern.program().build(q.device()).binary; + if (!any_of(type_equals(module::section::text_executable), m.secs)) + throw error(CL_INVALID_PROGRAM_EXECUTABLE); + } + + std::vector<size_t> + validate_grid_size(const command_queue &q, cl_uint dims, + const size_t *d_grid_size) { + auto grid_size = range(d_grid_size, dims); + + if (dims < 1 || dims > q.device().max_block_size().size()) + throw error(CL_INVALID_WORK_DIMENSION); + + if (!d_grid_size || any_of(is_zero(), grid_size)) + throw error(CL_INVALID_GLOBAL_WORK_SIZE); + + return grid_size; + } + + std::vector<size_t> + validate_grid_offset(const command_queue &q, cl_uint dims, + const size_t *d_grid_offset) { + if (d_grid_offset) + return range(d_grid_offset, dims); + else + return std::vector<size_t>(dims, 0); + } + + std::vector<size_t> + validate_block_size(const command_queue &q, const kernel &kern, + cl_uint dims, const size_t *d_grid_size, + const size_t *d_block_size) { + auto grid_size = range(d_grid_size, dims); + + if (d_block_size) { + auto block_size = range(d_block_size, dims); + + if (any_of(is_zero(), block_size) || + any_of(greater(), block_size, q.device().max_block_size())) + throw error(CL_INVALID_WORK_ITEM_SIZE); + + if (any_of(modulus(), grid_size, block_size)) + throw error(CL_INVALID_WORK_GROUP_SIZE); + + if (fold(multiplies(), 1u, block_size) > + q.device().max_threads_per_block()) + throw error(CL_INVALID_WORK_GROUP_SIZE); + + return block_size; + + } else { + return kern.optimal_block_size(q, grid_size); + } + } +} + +CLOVER_API cl_int +clEnqueueNDRangeKernel(cl_command_queue d_q, cl_kernel d_kern, + cl_uint dims, const size_t *d_grid_offset, + const size_t *d_grid_size, const size_t *d_block_size, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &kern = obj(d_kern); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto grid_size = validate_grid_size(q, dims, d_grid_size); + auto grid_offset = validate_grid_offset(q, dims, d_grid_offset); + auto block_size = validate_block_size(q, kern, dims, + d_grid_size, d_block_size); + + validate_common(q, kern, deps); + + auto hev = create<hard_event>( + q, CL_COMMAND_NDRANGE_KERNEL, deps, + [=, &kern, &q](event &) { + kern.launch(q, grid_offset, grid_size, block_size); + }); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueTask(cl_command_queue d_q, cl_kernel d_kern, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &kern = obj(d_kern); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + + validate_common(q, kern, deps); + + auto hev = create<hard_event>( + q, CL_COMMAND_TASK, deps, + [=, &kern, &q](event &) { + kern.launch(q, { 0 }, { 1 }, { 1 }); + }); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueNativeKernel(cl_command_queue d_q, void (*func)(void *), + void *args, size_t args_size, + cl_uint num_mems, const cl_mem *d_mems, + const void **mem_handles, cl_uint num_deps, + const cl_event *d_deps, cl_event *rd_ev) { + return CL_INVALID_OPERATION; +} + +CLOVER_API cl_int +clSetKernelArgSVMPointer(cl_kernel d_kern, + cl_uint arg_index, + const void *arg_value) try { + obj(d_kern).args().at(arg_index).set_svm(arg_value); + return CL_SUCCESS; + +} catch (std::out_of_range &e) { + return CL_INVALID_ARG_INDEX; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clSetKernelExecInfo(cl_kernel d_kern, + cl_kernel_exec_info param_name, + size_t param_value_size, + const void *param_value) try { + auto &kern = obj(d_kern); + const bool has_system_svm = all_of(std::mem_fn(&device::has_system_svm), + kern.program().context().devices()); + + if (!param_value) + return CL_INVALID_VALUE; + + switch (param_name) { + case CL_KERNEL_EXEC_INFO_SVM_FINE_GRAIN_SYSTEM: + case CL_KERNEL_EXEC_INFO_SVM_FINE_GRAIN_SYSTEM_ARM: { + if (param_value_size != sizeof(cl_bool)) + return CL_INVALID_VALUE; + + cl_bool val = *static_cast<const cl_bool*>(param_value); + if (val == CL_TRUE && !has_system_svm) + return CL_INVALID_OPERATION; + else + return CL_SUCCESS; + } + + case CL_KERNEL_EXEC_INFO_SVM_PTRS: + case CL_KERNEL_EXEC_INFO_SVM_PTRS_ARM: + if (has_system_svm) + return CL_SUCCESS; + + CLOVER_NOT_SUPPORTED_UNTIL("2.0"); + return CL_INVALID_VALUE; + + default: + return CL_INVALID_VALUE; + } + +} catch (error &e) { + return e.get(); +} diff --git a/src/gallium/frontends/clover/api/memory.cpp b/src/gallium/frontends/clover/api/memory.cpp new file mode 100644 index 00000000000..e03793339c1 --- /dev/null +++ b/src/gallium/frontends/clover/api/memory.cpp @@ -0,0 +1,497 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "util/u_math.h" +#include "api/util.hpp" +#include "core/memory.hpp" +#include "core/format.hpp" + +using namespace clover; + +namespace { + cl_mem_flags + validate_flags(cl_mem d_parent, cl_mem_flags d_flags, bool svm) { + const cl_mem_flags dev_access_flags = + CL_MEM_READ_WRITE | CL_MEM_WRITE_ONLY | CL_MEM_READ_ONLY; + const cl_mem_flags host_ptr_flags = + CL_MEM_USE_HOST_PTR | CL_MEM_ALLOC_HOST_PTR | CL_MEM_COPY_HOST_PTR; + const cl_mem_flags host_access_flags = + CL_MEM_HOST_WRITE_ONLY | CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_NO_ACCESS; + const cl_mem_flags svm_flags = + CL_MEM_SVM_FINE_GRAIN_BUFFER | CL_MEM_SVM_ATOMICS; + + const cl_mem_flags valid_flags = + dev_access_flags + | (svm || d_parent ? 0 : host_ptr_flags) + | (svm ? svm_flags : host_access_flags); + + if ((d_flags & ~valid_flags) || + util_bitcount(d_flags & dev_access_flags) > 1 || + util_bitcount(d_flags & host_access_flags) > 1) + throw error(CL_INVALID_VALUE); + + if ((d_flags & CL_MEM_USE_HOST_PTR) && + (d_flags & (CL_MEM_COPY_HOST_PTR | CL_MEM_ALLOC_HOST_PTR))) + throw error(CL_INVALID_VALUE); + + if ((d_flags & CL_MEM_SVM_ATOMICS) && + !(d_flags & CL_MEM_SVM_FINE_GRAIN_BUFFER)) + throw error(CL_INVALID_VALUE); + + if (d_parent) { + const auto &parent = obj(d_parent); + const cl_mem_flags flags = (d_flags | + (d_flags & dev_access_flags ? 0 : + parent.flags() & dev_access_flags) | + (d_flags & host_access_flags ? 0 : + parent.flags() & host_access_flags) | + (parent.flags() & host_ptr_flags)); + + if (~flags & parent.flags() & (dev_access_flags & ~CL_MEM_READ_WRITE)) + throw error(CL_INVALID_VALUE); + + // Check if new host access flags cause a mismatch between + // host-read/write-only. + if (!(flags & CL_MEM_HOST_NO_ACCESS) && + (~flags & parent.flags() & host_access_flags)) + throw error(CL_INVALID_VALUE); + + return flags; + + } else { + return d_flags | (d_flags & dev_access_flags ? 0 : CL_MEM_READ_WRITE); + } + } +} + +CLOVER_API cl_mem +clCreateBuffer(cl_context d_ctx, cl_mem_flags d_flags, size_t size, + void *host_ptr, cl_int *r_errcode) try { + const cl_mem_flags flags = validate_flags(NULL, d_flags, false); + auto &ctx = obj(d_ctx); + + if (bool(host_ptr) != bool(flags & (CL_MEM_USE_HOST_PTR | + CL_MEM_COPY_HOST_PTR))) + throw error(CL_INVALID_HOST_PTR); + + if (!size || + size > fold(maximum(), cl_ulong(0), + map(std::mem_fn(&device::max_mem_alloc_size), ctx.devices()) + )) + throw error(CL_INVALID_BUFFER_SIZE); + + ret_error(r_errcode, CL_SUCCESS); + return new root_buffer(ctx, flags, size, host_ptr); + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_mem +clCreateSubBuffer(cl_mem d_mem, cl_mem_flags d_flags, + cl_buffer_create_type op, + const void *op_info, cl_int *r_errcode) try { + auto &parent = obj<root_buffer>(d_mem); + const cl_mem_flags flags = validate_flags(d_mem, d_flags, false); + + if (op == CL_BUFFER_CREATE_TYPE_REGION) { + auto reg = reinterpret_cast<const cl_buffer_region *>(op_info); + + if (!reg || + reg->origin > parent.size() || + reg->origin + reg->size > parent.size()) + throw error(CL_INVALID_VALUE); + + if (!reg->size) + throw error(CL_INVALID_BUFFER_SIZE); + + ret_error(r_errcode, CL_SUCCESS); + return new sub_buffer(parent, flags, reg->origin, reg->size); + + } else { + throw error(CL_INVALID_VALUE); + } + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_mem +clCreateImage(cl_context d_ctx, cl_mem_flags d_flags, + const cl_image_format *format, + const cl_image_desc *desc, + void *host_ptr, cl_int *r_errcode) try { + auto &ctx = obj(d_ctx); + + if (!any_of(std::mem_fn(&device::image_support), ctx.devices())) + throw error(CL_INVALID_OPERATION); + + if (!format) + throw error(CL_INVALID_IMAGE_FORMAT_DESCRIPTOR); + + if (!desc) + throw error(CL_INVALID_IMAGE_DESCRIPTOR); + + if (desc->image_array_size == 0 && + (desc->image_type == CL_MEM_OBJECT_IMAGE1D_ARRAY || + desc->image_type == CL_MEM_OBJECT_IMAGE2D_ARRAY)) + throw error(CL_INVALID_IMAGE_DESCRIPTOR); + + if (!host_ptr && + (desc->image_row_pitch || desc->image_slice_pitch)) + throw error(CL_INVALID_IMAGE_DESCRIPTOR); + + if (desc->num_mip_levels || desc->num_samples) + throw error(CL_INVALID_IMAGE_DESCRIPTOR); + + if (bool(desc->buffer) != (desc->image_type == CL_MEM_OBJECT_IMAGE1D_BUFFER)) + throw error(CL_INVALID_IMAGE_DESCRIPTOR); + + if (bool(host_ptr) != bool(d_flags & (CL_MEM_USE_HOST_PTR | + CL_MEM_COPY_HOST_PTR))) + throw error(CL_INVALID_HOST_PTR); + + const cl_mem_flags flags = validate_flags(desc->buffer, d_flags, false); + + if (!supported_formats(ctx, desc->image_type).count(*format)) + throw error(CL_IMAGE_FORMAT_NOT_SUPPORTED); + + ret_error(r_errcode, CL_SUCCESS); + + switch (desc->image_type) { + case CL_MEM_OBJECT_IMAGE2D: + if (!desc->image_width || !desc->image_height) + throw error(CL_INVALID_IMAGE_SIZE); + + if (all_of([=](const device &dev) { + const size_t max = 1 << dev.max_image_levels_2d(); + return (desc->image_width > max || + desc->image_height > max); + }, ctx.devices())) + throw error(CL_INVALID_IMAGE_SIZE); + + return new image2d(ctx, flags, format, + desc->image_width, desc->image_height, + desc->image_row_pitch, host_ptr); + + case CL_MEM_OBJECT_IMAGE3D: + if (!desc->image_width || !desc->image_height || !desc->image_depth) + throw error(CL_INVALID_IMAGE_SIZE); + + if (all_of([=](const device &dev) { + const size_t max = 1 << dev.max_image_levels_3d(); + return (desc->image_width > max || + desc->image_height > max || + desc->image_depth > max); + }, ctx.devices())) + throw error(CL_INVALID_IMAGE_SIZE); + + return new image3d(ctx, flags, format, + desc->image_width, desc->image_height, + desc->image_depth, desc->image_row_pitch, + desc->image_slice_pitch, host_ptr); + + case CL_MEM_OBJECT_IMAGE1D: + case CL_MEM_OBJECT_IMAGE1D_ARRAY: + case CL_MEM_OBJECT_IMAGE1D_BUFFER: + case CL_MEM_OBJECT_IMAGE2D_ARRAY: + // XXX - Not implemented. + throw error(CL_IMAGE_FORMAT_NOT_SUPPORTED); + + default: + throw error(CL_INVALID_IMAGE_DESCRIPTOR); + } + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_mem +clCreateImage2D(cl_context d_ctx, cl_mem_flags d_flags, + const cl_image_format *format, + size_t width, size_t height, size_t row_pitch, + void *host_ptr, cl_int *r_errcode) { + const cl_image_desc desc = { CL_MEM_OBJECT_IMAGE2D, width, height, 0, 0, + row_pitch, 0, 0, 0, NULL }; + + return clCreateImage(d_ctx, d_flags, format, &desc, host_ptr, r_errcode); +} + +CLOVER_API cl_mem +clCreateImage3D(cl_context d_ctx, cl_mem_flags d_flags, + const cl_image_format *format, + size_t width, size_t height, size_t depth, + size_t row_pitch, size_t slice_pitch, + void *host_ptr, cl_int *r_errcode) { + const cl_image_desc desc = { CL_MEM_OBJECT_IMAGE3D, width, height, depth, 0, + row_pitch, slice_pitch, 0, 0, NULL }; + + return clCreateImage(d_ctx, d_flags, format, &desc, host_ptr, r_errcode); +} + +CLOVER_API cl_int +clGetSupportedImageFormats(cl_context d_ctx, cl_mem_flags flags, + cl_mem_object_type type, cl_uint count, + cl_image_format *r_buf, cl_uint *r_count) try { + auto &ctx = obj(d_ctx); + auto formats = supported_formats(ctx, type); + + validate_flags(NULL, flags, false); + + if (r_buf && !r_count) + throw error(CL_INVALID_VALUE); + + if (r_buf) + std::copy_n(formats.begin(), + std::min((cl_uint)formats.size(), count), + r_buf); + + if (r_count) + *r_count = formats.size(); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetMemObjectInfo(cl_mem d_mem, cl_mem_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &mem = obj(d_mem); + + switch (param) { + case CL_MEM_TYPE: + buf.as_scalar<cl_mem_object_type>() = mem.type(); + break; + + case CL_MEM_FLAGS: + buf.as_scalar<cl_mem_flags>() = mem.flags(); + break; + + case CL_MEM_SIZE: + buf.as_scalar<size_t>() = mem.size(); + break; + + case CL_MEM_HOST_PTR: + buf.as_scalar<void *>() = mem.host_ptr(); + break; + + case CL_MEM_MAP_COUNT: + buf.as_scalar<cl_uint>() = 0; + break; + + case CL_MEM_REFERENCE_COUNT: + buf.as_scalar<cl_uint>() = mem.ref_count(); + break; + + case CL_MEM_CONTEXT: + buf.as_scalar<cl_context>() = desc(mem.context()); + break; + + case CL_MEM_ASSOCIATED_MEMOBJECT: { + sub_buffer *sub = dynamic_cast<sub_buffer *>(&mem); + buf.as_scalar<cl_mem>() = (sub ? desc(sub->parent()) : NULL); + break; + } + case CL_MEM_OFFSET: { + sub_buffer *sub = dynamic_cast<sub_buffer *>(&mem); + buf.as_scalar<size_t>() = (sub ? sub->offset() : 0); + break; + } + case CL_MEM_USES_SVM_POINTER: + case CL_MEM_USES_SVM_POINTER_ARM: { + // with system SVM all host ptrs are SVM pointers + // TODO: once we support devices with lower levels of SVM, we have to + // check the ptr in more detail + const bool system_svm = all_of(std::mem_fn(&device::has_system_svm), + mem.context().devices()); + buf.as_scalar<cl_bool>() = mem.host_ptr() && system_svm; + break; + } + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetImageInfo(cl_mem d_mem, cl_image_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &img = obj<image>(d_mem); + + switch (param) { + case CL_IMAGE_FORMAT: + buf.as_scalar<cl_image_format>() = img.format(); + break; + + case CL_IMAGE_ELEMENT_SIZE: + buf.as_scalar<size_t>() = 0; + break; + + case CL_IMAGE_ROW_PITCH: + buf.as_scalar<size_t>() = img.row_pitch(); + break; + + case CL_IMAGE_SLICE_PITCH: + buf.as_scalar<size_t>() = img.slice_pitch(); + break; + + case CL_IMAGE_WIDTH: + buf.as_scalar<size_t>() = img.width(); + break; + + case CL_IMAGE_HEIGHT: + buf.as_scalar<size_t>() = img.height(); + break; + + case CL_IMAGE_DEPTH: + buf.as_scalar<size_t>() = img.depth(); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clRetainMemObject(cl_mem d_mem) try { + obj(d_mem).retain(); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clReleaseMemObject(cl_mem d_mem) try { + if (obj(d_mem).release()) + delete pobj(d_mem); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clSetMemObjectDestructorCallback(cl_mem d_mem, + void (CL_CALLBACK *pfn_notify)(cl_mem, void *), + void *user_data) try { + auto &mem = obj(d_mem); + + if (!pfn_notify) + return CL_INVALID_VALUE; + + mem.destroy_notify([=]{ pfn_notify(d_mem, user_data); }); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueFillBuffer(cl_command_queue command_queue, cl_mem buffer, + const void *pattern, size_t pattern_size, + size_t offset, size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + CLOVER_NOT_SUPPORTED_UNTIL("1.2"); + return CL_INVALID_VALUE; +} + +CLOVER_API cl_int +clEnqueueFillImage(cl_command_queue command_queue, cl_mem image, + const void *fill_color, + const size_t *origin, const size_t *region, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + CLOVER_NOT_SUPPORTED_UNTIL("1.2"); + return CL_INVALID_VALUE; +} + +CLOVER_API void * +clSVMAlloc(cl_context d_ctx, + cl_svm_mem_flags flags, + size_t size, + unsigned int alignment) try { + auto &ctx = obj(d_ctx); + validate_flags(NULL, flags, true); + + if (!size || + size > fold(minimum(), cl_ulong(ULONG_MAX), + map(std::mem_fn(&device::max_mem_alloc_size), ctx.devices()))) + return nullptr; + + if (!util_is_power_of_two_or_zero(alignment)) + return nullptr; + + if (!alignment) + alignment = 0x80; // sizeof(long16) + + bool can_emulate = all_of(std::mem_fn(&device::has_system_svm), ctx.devices()); + if (can_emulate) { + // we can ignore all the flags as it's not required to honor them. + void *ptr = nullptr; + if (alignment < sizeof(void*)) + alignment = sizeof(void*); + posix_memalign(&ptr, alignment, size); + return ptr; + } + + CLOVER_NOT_SUPPORTED_UNTIL("2.0"); + return nullptr; + +} catch (error &e) { + return nullptr; +} + +CLOVER_API void +clSVMFree(cl_context d_ctx, + void *svm_pointer) try { + auto &ctx = obj(d_ctx); + bool can_emulate = all_of(std::mem_fn(&device::has_system_svm), ctx.devices()); + + if (can_emulate) + return free(svm_pointer); + + CLOVER_NOT_SUPPORTED_UNTIL("2.0"); + +} catch (error &e) { +} diff --git a/src/gallium/frontends/clover/api/platform.cpp b/src/gallium/frontends/clover/api/platform.cpp new file mode 100644 index 00000000000..7360461e62f --- /dev/null +++ b/src/gallium/frontends/clover/api/platform.cpp @@ -0,0 +1,235 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include <unordered_map> + +#include "api/dispatch.hpp" +#include "api/util.hpp" +#include "core/platform.hpp" +#include "git_sha1.h" +#include "util/u_debug.h" + +using namespace clover; + +namespace { + platform _clover_platform; +} + +CLOVER_API cl_int +clGetPlatformIDs(cl_uint num_entries, cl_platform_id *rd_platforms, + cl_uint *rnum_platforms) { + if ((!num_entries && rd_platforms) || + (!rnum_platforms && !rd_platforms)) + return CL_INVALID_VALUE; + + if (rnum_platforms) + *rnum_platforms = 1; + if (rd_platforms) + *rd_platforms = desc(_clover_platform); + + return CL_SUCCESS; +} + +cl_int +clover::GetPlatformInfo(cl_platform_id d_platform, cl_platform_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + + auto &platform = obj(d_platform); + + switch (param) { + case CL_PLATFORM_PROFILE: + buf.as_string() = "FULL_PROFILE"; + break; + + case CL_PLATFORM_VERSION: { + static const std::string version_string = + debug_get_option("CLOVER_PLATFORM_VERSION_OVERRIDE", "1.1"); + + buf.as_string() = "OpenCL " + version_string + " Mesa " PACKAGE_VERSION MESA_GIT_SHA1; + break; + } + case CL_PLATFORM_NAME: + buf.as_string() = "Clover"; + break; + + case CL_PLATFORM_VENDOR: + buf.as_string() = "Mesa"; + break; + + case CL_PLATFORM_EXTENSIONS: + buf.as_string() = platform.supported_extensions(); + break; + + case CL_PLATFORM_ICD_SUFFIX_KHR: + buf.as_string() = "MESA"; + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +void * +clover::GetExtensionFunctionAddressForPlatform(cl_platform_id d_platform, + const char *p_name) try { + obj(d_platform); + return GetExtensionFunctionAddress(p_name); + +} catch (error &e) { + return NULL; +} + +namespace { + +cl_int +enqueueSVMFreeARM(cl_command_queue command_queue, + cl_uint num_svm_pointers, + void *svm_pointers[], + void (CL_CALLBACK *pfn_free_func) ( + cl_command_queue queue, cl_uint num_svm_pointers, + void *svm_pointers[], void *user_data), + void *user_data, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMFree(command_queue, num_svm_pointers, svm_pointers, + pfn_free_func, user_data, num_events_in_wait_list, + event_wait_list, event, CL_COMMAND_SVM_FREE_ARM); +} + +cl_int +enqueueSVMMapARM(cl_command_queue command_queue, + cl_bool blocking_map, + cl_map_flags map_flags, + void *svm_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMMap(command_queue, blocking_map, map_flags, svm_ptr, size, + num_events_in_wait_list, event_wait_list, event, + CL_COMMAND_SVM_MAP_ARM); +} + +cl_int +enqueueSVMMemcpyARM(cl_command_queue command_queue, + cl_bool blocking_copy, + void *dst_ptr, + const void *src_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMMemcpy(command_queue, blocking_copy, dst_ptr, src_ptr, + size, num_events_in_wait_list, event_wait_list, + event, CL_COMMAND_SVM_MEMCPY_ARM); +} + +cl_int +enqueueSVMMemFillARM(cl_command_queue command_queue, + void *svm_ptr, + const void *pattern, + size_t pattern_size, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMMemFill(command_queue, svm_ptr, pattern, pattern_size, + size, num_events_in_wait_list, event_wait_list, + event, CL_COMMAND_SVM_MEMFILL_ARM); +} + +cl_int +enqueueSVMUnmapARM(cl_command_queue command_queue, + void *svm_ptr, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMUnmap(command_queue, svm_ptr, num_events_in_wait_list, + event_wait_list, event, CL_COMMAND_SVM_UNMAP_ARM); +} + +const std::unordered_map<std::string, void *> +ext_funcs = { + // cl_arm_shared_virtual_memory + { "clEnqueueSVMFreeARM", reinterpret_cast<void *>(enqueueSVMFreeARM) }, + { "clEnqueueSVMMapARM", reinterpret_cast<void *>(enqueueSVMMapARM) }, + { "clEnqueueSVMMemcpyARM", reinterpret_cast<void *>(enqueueSVMMemcpyARM) }, + { "clEnqueueSVMMemFillARM", reinterpret_cast<void *>(enqueueSVMMemFillARM) }, + { "clEnqueueSVMUnmapARM", reinterpret_cast<void *>(enqueueSVMUnmapARM) }, + { "clSetKernelArgSVMPointerARM", reinterpret_cast<void *>(clSetKernelArgSVMPointer) }, + { "clSetKernelExecInfoARM", reinterpret_cast<void *>(clSetKernelExecInfo) }, + { "clSVMAllocARM", reinterpret_cast<void *>(clSVMAlloc) }, + { "clSVMFreeARM", reinterpret_cast<void *>(clSVMFree) }, + + // cl_khr_icd + { "clIcdGetPlatformIDsKHR", reinterpret_cast<void *>(IcdGetPlatformIDsKHR) }, +}; + +} // anonymous namespace + +void * +clover::GetExtensionFunctionAddress(const char *p_name) try { + return ext_funcs.at(p_name); +} catch (...) { + return nullptr; +} + +cl_int +clover::IcdGetPlatformIDsKHR(cl_uint num_entries, cl_platform_id *rd_platforms, + cl_uint *rnum_platforms) { + return clGetPlatformIDs(num_entries, rd_platforms, rnum_platforms); +} + +CLOVER_ICD_API cl_int +clGetPlatformInfo(cl_platform_id d_platform, cl_platform_info param, + size_t size, void *r_buf, size_t *r_size) { + return GetPlatformInfo(d_platform, param, size, r_buf, r_size); +} + +CLOVER_ICD_API void * +clGetExtensionFunctionAddress(const char *p_name) { + return GetExtensionFunctionAddress(p_name); +} + +CLOVER_ICD_API void * +clGetExtensionFunctionAddressForPlatform(cl_platform_id d_platform, + const char *p_name) { + return GetExtensionFunctionAddressForPlatform(d_platform, p_name); +} + +CLOVER_ICD_API cl_int +clIcdGetPlatformIDsKHR(cl_uint num_entries, cl_platform_id *rd_platforms, + cl_uint *rnum_platforms) { + return IcdGetPlatformIDsKHR(num_entries, rd_platforms, rnum_platforms); +} diff --git a/src/gallium/frontends/clover/api/program.cpp b/src/gallium/frontends/clover/api/program.cpp new file mode 100644 index 00000000000..33f843e9c87 --- /dev/null +++ b/src/gallium/frontends/clover/api/program.cpp @@ -0,0 +1,479 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "api/util.hpp" +#include "core/program.hpp" +#include "util/u_debug.h" + +#include <sstream> + +using namespace clover; + +namespace { + void + validate_build_common(const program &prog, cl_uint num_devs, + const cl_device_id *d_devs, + void (*pfn_notify)(cl_program, void *), + void *user_data) { + if (!pfn_notify && user_data) + throw error(CL_INVALID_VALUE); + + if (prog.kernel_ref_count()) + throw error(CL_INVALID_OPERATION); + + if (any_of([&](const device &dev) { + return !count(dev, prog.devices()); + }, objs<allow_empty_tag>(d_devs, num_devs))) + throw error(CL_INVALID_DEVICE); + } +} + +CLOVER_API cl_program +clCreateProgramWithSource(cl_context d_ctx, cl_uint count, + const char **strings, const size_t *lengths, + cl_int *r_errcode) try { + auto &ctx = obj(d_ctx); + std::string source; + + if (!count || !strings || + any_of(is_zero(), range(strings, count))) + throw error(CL_INVALID_VALUE); + + // Concatenate all the provided fragments together + for (unsigned i = 0; i < count; ++i) + source += (lengths && lengths[i] ? + std::string(strings[i], strings[i] + lengths[i]) : + std::string(strings[i])); + + // ...and create a program object for them. + ret_error(r_errcode, CL_SUCCESS); + return new program(ctx, source); + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_program +clCreateProgramWithBinary(cl_context d_ctx, cl_uint n, + const cl_device_id *d_devs, + const size_t *lengths, + const unsigned char **binaries, + cl_int *r_status, cl_int *r_errcode) try { + auto &ctx = obj(d_ctx); + auto devs = objs(d_devs, n); + + if (!lengths || !binaries) + throw error(CL_INVALID_VALUE); + + if (any_of([&](const device &dev) { + return !count(dev, ctx.devices()); + }, devs)) + throw error(CL_INVALID_DEVICE); + + // Deserialize the provided binaries, + std::vector<std::pair<cl_int, module>> result = map( + [](const unsigned char *p, size_t l) -> std::pair<cl_int, module> { + if (!p || !l) + return { CL_INVALID_VALUE, {} }; + + try { + std::stringbuf bin( { (char*)p, l } ); + std::istream s(&bin); + + return { CL_SUCCESS, module::deserialize(s) }; + + } catch (std::istream::failure &e) { + return { CL_INVALID_BINARY, {} }; + } + }, + range(binaries, n), + range(lengths, n)); + + // update the status array, + if (r_status) + copy(map(keys(), result), r_status); + + if (any_of(key_equals(CL_INVALID_VALUE), result)) + throw error(CL_INVALID_VALUE); + + if (any_of(key_equals(CL_INVALID_BINARY), result)) + throw error(CL_INVALID_BINARY); + + // initialize a program object with them. + ret_error(r_errcode, CL_SUCCESS); + return new program(ctx, devs, map(values(), result)); + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_program +clCreateProgramWithBuiltInKernels(cl_context d_ctx, cl_uint n, + const cl_device_id *d_devs, + const char *kernel_names, + cl_int *r_errcode) try { + auto &ctx = obj(d_ctx); + auto devs = objs(d_devs, n); + + if (any_of([&](const device &dev) { + return !count(dev, ctx.devices()); + }, devs)) + throw error(CL_INVALID_DEVICE); + + // No currently supported built-in kernels. + throw error(CL_INVALID_VALUE); + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + + +CLOVER_API cl_int +clRetainProgram(cl_program d_prog) try { + obj(d_prog).retain(); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clReleaseProgram(cl_program d_prog) try { + if (obj(d_prog).release()) + delete pobj(d_prog); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clBuildProgram(cl_program d_prog, cl_uint num_devs, + const cl_device_id *d_devs, const char *p_opts, + void (*pfn_notify)(cl_program, void *), + void *user_data) try { + auto &prog = obj(d_prog); + auto devs = + (d_devs ? objs(d_devs, num_devs) : ref_vector<device>(prog.devices())); + const auto opts = std::string(p_opts ? p_opts : "") + " " + + debug_get_option("CLOVER_EXTRA_BUILD_OPTIONS", ""); + + validate_build_common(prog, num_devs, d_devs, pfn_notify, user_data); + + if (prog.has_source) { + prog.compile(devs, opts); + prog.link(devs, opts, { prog }); + } else if (any_of([&](const device &dev){ + return prog.build(dev).binary_type() != CL_PROGRAM_BINARY_TYPE_EXECUTABLE; + }, devs)) { + // According to the OpenCL 1.2 specification, “if program is created + // with clCreateProgramWithBinary, then the program binary must be an + // executable binary (not a compiled binary or library).” + throw error(CL_INVALID_BINARY); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clCompileProgram(cl_program d_prog, cl_uint num_devs, + const cl_device_id *d_devs, const char *p_opts, + cl_uint num_headers, const cl_program *d_header_progs, + const char **header_names, + void (*pfn_notify)(cl_program, void *), + void *user_data) try { + auto &prog = obj(d_prog); + auto devs = + (d_devs ? objs(d_devs, num_devs) : ref_vector<device>(prog.devices())); + const auto opts = std::string(p_opts ? p_opts : "") + " " + + debug_get_option("CLOVER_EXTRA_COMPILE_OPTIONS", ""); + header_map headers; + + validate_build_common(prog, num_devs, d_devs, pfn_notify, user_data); + + if (bool(num_headers) != bool(header_names)) + throw error(CL_INVALID_VALUE); + + if (!prog.has_source) + throw error(CL_INVALID_OPERATION); + + for_each([&](const char *name, const program &header) { + if (!header.has_source) + throw error(CL_INVALID_OPERATION); + + if (!any_of(key_equals(name), headers)) + headers.push_back(std::pair<std::string, std::string>( + name, header.source())); + }, + range(header_names, num_headers), + objs<allow_empty_tag>(d_header_progs, num_headers)); + + prog.compile(devs, opts, headers); + return CL_SUCCESS; + +} catch (invalid_build_options_error &e) { + return CL_INVALID_COMPILER_OPTIONS; + +} catch (build_error &e) { + return CL_COMPILE_PROGRAM_FAILURE; + +} catch (error &e) { + return e.get(); +} + +namespace { + ref_vector<device> + validate_link_devices(const ref_vector<program> &progs, + const ref_vector<device> &all_devs, + const std::string &opts) { + std::vector<device *> devs; + const bool create_library = + opts.find("-create-library") != std::string::npos; + const bool enable_link_options = + opts.find("-enable-link-options") != std::string::npos; + const bool has_link_options = + opts.find("-cl-denorms-are-zero") != std::string::npos || + opts.find("-cl-no-signed-zeroes") != std::string::npos || + opts.find("-cl-unsafe-math-optimizations") != std::string::npos || + opts.find("-cl-finite-math-only") != std::string::npos || + opts.find("-cl-fast-relaxed-math") != std::string::npos || + opts.find("-cl-no-subgroup-ifp") != std::string::npos; + + // According to the OpenCL 1.2 specification, "[the + // -enable-link-options] option must be specified with the + // create-library option". + if (enable_link_options && !create_library) + throw error(CL_INVALID_LINKER_OPTIONS); + + // According to the OpenCL 1.2 specification, "the + // [program linking options] can be specified when linking a program + // executable". + if (has_link_options && create_library) + throw error(CL_INVALID_LINKER_OPTIONS); + + for (auto &dev : all_devs) { + const auto has_binary = [&](const program &prog) { + const auto t = prog.build(dev).binary_type(); + return t == CL_PROGRAM_BINARY_TYPE_COMPILED_OBJECT || + t == CL_PROGRAM_BINARY_TYPE_LIBRARY; + }; + + // According to the OpenCL 1.2 specification, a library is made of + // “compiled binaries specified in input_programs argument to + // clLinkProgram“; compiled binaries does not refer to libraries: + // “input_programs is an array of program objects that are compiled + // binaries or libraries that are to be linked to create the program + // executable”. + if (create_library && any_of([&](const program &prog) { + const auto t = prog.build(dev).binary_type(); + return t != CL_PROGRAM_BINARY_TYPE_COMPILED_OBJECT; + }, progs)) + throw error(CL_INVALID_OPERATION); + + // According to the CL 1.2 spec, when "all programs specified [..] + // contain a compiled binary or library for the device [..] a link is + // performed", + else if (all_of(has_binary, progs)) + devs.push_back(&dev); + + // otherwise if "none of the programs contain a compiled binary or + // library for that device [..] no link is performed. All other + // cases will return a CL_INVALID_OPERATION error." + else if (any_of(has_binary, progs)) + throw error(CL_INVALID_OPERATION); + + // According to the OpenCL 1.2 specification, "[t]he linker may apply + // [program linking options] to all compiled program objects + // specified to clLinkProgram. The linker may apply these options + // only to libraries which were created with the + // -enable-link-option." + else if (has_link_options && any_of([&](const program &prog) { + const auto t = prog.build(dev).binary_type(); + return !(t == CL_PROGRAM_BINARY_TYPE_COMPILED_OBJECT || + (t == CL_PROGRAM_BINARY_TYPE_LIBRARY && + prog.build(dev).opts.find("-enable-link-options") != + std::string::npos)); + }, progs)) + throw error(CL_INVALID_LINKER_OPTIONS); + } + + return map(derefs(), devs); + } +} + +CLOVER_API cl_program +clLinkProgram(cl_context d_ctx, cl_uint num_devs, const cl_device_id *d_devs, + const char *p_opts, cl_uint num_progs, const cl_program *d_progs, + void (*pfn_notify) (cl_program, void *), void *user_data, + cl_int *r_errcode) try { + auto &ctx = obj(d_ctx); + const auto opts = std::string(p_opts ? p_opts : "") + " " + + debug_get_option("CLOVER_EXTRA_LINK_OPTIONS", ""); + auto progs = objs(d_progs, num_progs); + auto all_devs = + (d_devs ? objs(d_devs, num_devs) : ref_vector<device>(ctx.devices())); + auto prog = create<program>(ctx, all_devs); + auto devs = validate_link_devices(progs, all_devs, opts); + + validate_build_common(prog, num_devs, d_devs, pfn_notify, user_data); + + try { + prog().link(devs, opts, progs); + ret_error(r_errcode, CL_SUCCESS); + + } catch (build_error &e) { + ret_error(r_errcode, CL_LINK_PROGRAM_FAILURE); + } + + return ret_object(prog); + +} catch (invalid_build_options_error &e) { + ret_error(r_errcode, CL_INVALID_LINKER_OPTIONS); + return NULL; + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_int +clUnloadCompiler() { + return CL_SUCCESS; +} + +CLOVER_API cl_int +clUnloadPlatformCompiler(cl_platform_id d_platform) { + return CL_SUCCESS; +} + +CLOVER_API cl_int +clGetProgramInfo(cl_program d_prog, cl_program_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &prog = obj(d_prog); + + switch (param) { + case CL_PROGRAM_REFERENCE_COUNT: + buf.as_scalar<cl_uint>() = prog.ref_count(); + break; + + case CL_PROGRAM_CONTEXT: + buf.as_scalar<cl_context>() = desc(prog.context()); + break; + + case CL_PROGRAM_NUM_DEVICES: + buf.as_scalar<cl_uint>() = (prog.devices().size() ? + prog.devices().size() : + prog.context().devices().size()); + break; + + case CL_PROGRAM_DEVICES: + buf.as_vector<cl_device_id>() = (prog.devices().size() ? + descs(prog.devices()) : + descs(prog.context().devices())); + break; + + case CL_PROGRAM_SOURCE: + buf.as_string() = prog.source(); + break; + + case CL_PROGRAM_BINARY_SIZES: + buf.as_vector<size_t>() = map([&](const device &dev) { + return prog.build(dev).binary.size(); + }, + prog.devices()); + break; + + case CL_PROGRAM_BINARIES: + buf.as_matrix<unsigned char>() = map([&](const device &dev) { + std::stringbuf bin; + std::ostream s(&bin); + prog.build(dev).binary.serialize(s); + return bin.str(); + }, + prog.devices()); + break; + + case CL_PROGRAM_NUM_KERNELS: + buf.as_scalar<cl_uint>() = prog.symbols().size(); + break; + + case CL_PROGRAM_KERNEL_NAMES: + buf.as_string() = fold([](const std::string &a, const module::symbol &s) { + return ((a.empty() ? "" : a + ";") + s.name); + }, std::string(), prog.symbols()); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetProgramBuildInfo(cl_program d_prog, cl_device_id d_dev, + cl_program_build_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &prog = obj(d_prog); + auto &dev = obj(d_dev); + + if (!count(dev, prog.context().devices())) + return CL_INVALID_DEVICE; + + switch (param) { + case CL_PROGRAM_BUILD_STATUS: + buf.as_scalar<cl_build_status>() = prog.build(dev).status(); + break; + + case CL_PROGRAM_BUILD_OPTIONS: + buf.as_string() = prog.build(dev).opts; + break; + + case CL_PROGRAM_BUILD_LOG: + buf.as_string() = prog.build(dev).log; + break; + + case CL_PROGRAM_BINARY_TYPE: + buf.as_scalar<cl_program_binary_type>() = prog.build(dev).binary_type(); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} diff --git a/src/gallium/frontends/clover/api/queue.cpp b/src/gallium/frontends/clover/api/queue.cpp new file mode 100644 index 00000000000..65b271b216f --- /dev/null +++ b/src/gallium/frontends/clover/api/queue.cpp @@ -0,0 +1,135 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "api/util.hpp" +#include "core/queue.hpp" + +using namespace clover; + +CLOVER_API cl_command_queue +clCreateCommandQueue(cl_context d_ctx, cl_device_id d_dev, + cl_command_queue_properties props, + cl_int *r_errcode) try { + auto &ctx = obj(d_ctx); + auto &dev = obj(d_dev); + + if (!count(dev, ctx.devices())) + throw error(CL_INVALID_DEVICE); + + if (props & ~(CL_QUEUE_OUT_OF_ORDER_EXEC_MODE_ENABLE | + CL_QUEUE_PROFILING_ENABLE)) + throw error(CL_INVALID_VALUE); + + ret_error(r_errcode, CL_SUCCESS); + return new command_queue(ctx, dev, props); + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_int +clRetainCommandQueue(cl_command_queue d_q) try { + obj(d_q).retain(); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clReleaseCommandQueue(cl_command_queue d_q) try { + auto &q = obj(d_q); + + q.flush(); + + if (q.release()) + delete pobj(d_q); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetCommandQueueInfo(cl_command_queue d_q, cl_command_queue_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &q = obj(d_q); + + switch (param) { + case CL_QUEUE_CONTEXT: + buf.as_scalar<cl_context>() = desc(q.context()); + break; + + case CL_QUEUE_DEVICE: + buf.as_scalar<cl_device_id>() = desc(q.device()); + break; + + case CL_QUEUE_REFERENCE_COUNT: + buf.as_scalar<cl_uint>() = q.ref_count(); + break; + + case CL_QUEUE_PROPERTIES: + buf.as_scalar<cl_command_queue_properties>() = q.properties(); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clFlush(cl_command_queue d_q) try { + obj(d_q).flush(); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_command_queue +clCreateCommandQueueWithProperties(cl_context context, cl_device_id device, + const cl_queue_properties *properties, + cl_int *errcode_ret) try { + cl_command_queue_properties props = 0; + if (properties) { + for (auto idx = 0; properties[idx]; idx += 2) { + if (properties[idx] == CL_QUEUE_PROPERTIES) + props |= properties[idx + 1]; + else + throw error(CL_INVALID_VALUE); + } + } + + return clCreateCommandQueue(context, device, props, errcode_ret); + +} catch (error &e) { + ret_error(errcode_ret, e); + return NULL; +} diff --git a/src/gallium/frontends/clover/api/sampler.cpp b/src/gallium/frontends/clover/api/sampler.cpp new file mode 100644 index 00000000000..482e55a9ce9 --- /dev/null +++ b/src/gallium/frontends/clover/api/sampler.cpp @@ -0,0 +1,100 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "api/util.hpp" +#include "core/sampler.hpp" + +using namespace clover; + +CLOVER_API cl_sampler +clCreateSampler(cl_context d_ctx, cl_bool norm_mode, + cl_addressing_mode addr_mode, cl_filter_mode filter_mode, + cl_int *r_errcode) try { + auto &ctx = obj(d_ctx); + + if (!any_of(std::mem_fn(&device::image_support), ctx.devices())) + throw error(CL_INVALID_OPERATION); + + ret_error(r_errcode, CL_SUCCESS); + return new sampler(ctx, norm_mode, addr_mode, filter_mode); + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_int +clRetainSampler(cl_sampler d_s) try { + obj(d_s).retain(); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clReleaseSampler(cl_sampler d_s) try { + if (obj(d_s).release()) + delete pobj(d_s); + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clGetSamplerInfo(cl_sampler d_s, cl_sampler_info param, + size_t size, void *r_buf, size_t *r_size) try { + property_buffer buf { r_buf, size, r_size }; + auto &s = obj(d_s); + + switch (param) { + case CL_SAMPLER_REFERENCE_COUNT: + buf.as_scalar<cl_uint>() = s.ref_count(); + break; + + case CL_SAMPLER_CONTEXT: + buf.as_scalar<cl_context>() = desc(s.context()); + break; + + case CL_SAMPLER_NORMALIZED_COORDS: + buf.as_scalar<cl_bool>() = s.norm_mode(); + break; + + case CL_SAMPLER_ADDRESSING_MODE: + buf.as_scalar<cl_addressing_mode>() = s.addr_mode(); + break; + + case CL_SAMPLER_FILTER_MODE: + buf.as_scalar<cl_filter_mode>() = s.filter_mode(); + break; + + default: + throw error(CL_INVALID_VALUE); + } + + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} diff --git a/src/gallium/frontends/clover/api/transfer.cpp b/src/gallium/frontends/clover/api/transfer.cpp new file mode 100644 index 00000000000..fa8741e02b4 --- /dev/null +++ b/src/gallium/frontends/clover/api/transfer.cpp @@ -0,0 +1,1059 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include <cstring> + +#include "util/bitscan.h" + +#include "api/dispatch.hpp" +#include "api/util.hpp" +#include "core/event.hpp" +#include "core/memory.hpp" + +using namespace clover; + +namespace { + typedef resource::vector vector_t; + + vector_t + vector(const size_t *p) { + return range(p, 3); + } + + vector_t + pitch(const vector_t ®ion, vector_t pitch) { + for (auto x : zip(tail(pitch), + map(multiplies(), region, pitch))) { + // The spec defines a value of zero as the natural pitch, + // i.e. the unaligned size of the previous dimension. + if (std::get<0>(x) == 0) + std::get<0>(x) = std::get<1>(x); + } + + return pitch; + } + + /// + /// Size of a region in bytes. + /// + size_t + size(const vector_t &pitch, const vector_t ®ion) { + if (any_of(is_zero(), region)) + return 0; + else + return dot(pitch, region - vector_t{ 0, 1, 1 }); + } + + /// + /// Common argument checking shared by memory transfer commands. + /// + void + validate_common(command_queue &q, + const ref_vector<event> &deps) { + if (any_of([&](const event &ev) { + return ev.context() != q.context(); + }, deps)) + throw error(CL_INVALID_CONTEXT); + } + + /// + /// Common error checking for a buffer object argument. + /// + void + validate_object(command_queue &q, buffer &mem, const vector_t &origin, + const vector_t &pitch, const vector_t ®ion) { + if (mem.context() != q.context()) + throw error(CL_INVALID_CONTEXT); + + // The region must fit within the specified pitch, + if (any_of(greater(), map(multiplies(), pitch, region), tail(pitch))) + throw error(CL_INVALID_VALUE); + + // ...and within the specified object. + if (dot(pitch, origin) + size(pitch, region) > mem.size()) + throw error(CL_INVALID_VALUE); + + if (any_of(is_zero(), region)) + throw error(CL_INVALID_VALUE); + } + + /// + /// Common error checking for an image argument. + /// + void + validate_object(command_queue &q, image &img, + const vector_t &orig, const vector_t ®ion) { + vector_t size = { img.width(), img.height(), img.depth() }; + + if (!q.device().image_support()) + throw error(CL_INVALID_OPERATION); + + if (img.context() != q.context()) + throw error(CL_INVALID_CONTEXT); + + if (any_of(greater(), orig + region, size)) + throw error(CL_INVALID_VALUE); + + if (any_of(is_zero(), region)) + throw error(CL_INVALID_VALUE); + } + + /// + /// Common error checking for a host pointer argument. + /// + void + validate_object(command_queue &q, const void *ptr, const vector_t &orig, + const vector_t &pitch, const vector_t ®ion) { + if (!ptr) + throw error(CL_INVALID_VALUE); + + // The region must fit within the specified pitch. + if (any_of(greater(), map(multiplies(), pitch, region), tail(pitch))) + throw error(CL_INVALID_VALUE); + } + + /// + /// Common argument checking for a copy between two buffer objects. + /// + void + validate_copy(command_queue &q, buffer &dst_mem, + const vector_t &dst_orig, const vector_t &dst_pitch, + buffer &src_mem, + const vector_t &src_orig, const vector_t &src_pitch, + const vector_t ®ion) { + if (dst_mem == src_mem) { + auto dst_offset = dot(dst_pitch, dst_orig); + auto src_offset = dot(src_pitch, src_orig); + + if (interval_overlaps()( + dst_offset, dst_offset + size(dst_pitch, region), + src_offset, src_offset + size(src_pitch, region))) + throw error(CL_MEM_COPY_OVERLAP); + } + } + + /// + /// Common argument checking for a copy between two image objects. + /// + void + validate_copy(command_queue &q, + image &dst_img, const vector_t &dst_orig, + image &src_img, const vector_t &src_orig, + const vector_t ®ion) { + if (dst_img.format() != src_img.format()) + throw error(CL_IMAGE_FORMAT_MISMATCH); + + if (dst_img == src_img) { + if (all_of(interval_overlaps(), + dst_orig, dst_orig + region, + src_orig, src_orig + region)) + throw error(CL_MEM_COPY_OVERLAP); + } + } + + /// + /// Checks that the host access flags of the memory object are + /// within the allowed set \a flags. + /// + void + validate_object_access(const memory_obj &mem, const cl_mem_flags flags) { + if (mem.flags() & ~flags & + (CL_MEM_HOST_READ_ONLY | CL_MEM_HOST_WRITE_ONLY | + CL_MEM_HOST_NO_ACCESS)) + throw error(CL_INVALID_OPERATION); + } + + /// + /// Checks that the mapping flags are correct. + /// + void + validate_map_flags(const memory_obj &mem, const cl_map_flags flags) { + if ((flags & (CL_MAP_WRITE | CL_MAP_READ)) && + (flags & CL_MAP_WRITE_INVALIDATE_REGION)) + throw error(CL_INVALID_VALUE); + + if (flags & CL_MAP_READ) + validate_object_access(mem, CL_MEM_HOST_READ_ONLY); + + if (flags & (CL_MAP_WRITE | CL_MAP_WRITE_INVALIDATE_REGION)) + validate_object_access(mem, CL_MEM_HOST_WRITE_ONLY); + } + + /// + /// Class that encapsulates the task of mapping an object of type + /// \a T. The return value of get() should be implicitly + /// convertible to \a void *. + /// + template<typename T> + struct _map { + static mapping + get(command_queue &q, T obj, cl_map_flags flags, + size_t offset, size_t size) { + return { q, obj->resource(q), flags, true, + {{ offset }}, {{ size, 1, 1 }} }; + } + }; + + template<> + struct _map<void *> { + static void * + get(command_queue &q, void *obj, cl_map_flags flags, + size_t offset, size_t size) { + return (char *)obj + offset; + } + }; + + template<> + struct _map<const void *> { + static const void * + get(command_queue &q, const void *obj, cl_map_flags flags, + size_t offset, size_t size) { + return (const char *)obj + offset; + } + }; + + /// + /// Software copy from \a src_obj to \a dst_obj. They can be + /// either pointers or memory objects. + /// + template<typename T, typename S> + std::function<void (event &)> + soft_copy_op(command_queue &q, + T dst_obj, const vector_t &dst_orig, const vector_t &dst_pitch, + S src_obj, const vector_t &src_orig, const vector_t &src_pitch, + const vector_t ®ion) { + return [=, &q](event &) { + auto dst = _map<T>::get(q, dst_obj, CL_MAP_WRITE, + dot(dst_pitch, dst_orig), + size(dst_pitch, region)); + auto src = _map<S>::get(q, src_obj, CL_MAP_READ, + dot(src_pitch, src_orig), + size(src_pitch, region)); + vector_t v = {}; + + for (v[2] = 0; v[2] < region[2]; ++v[2]) { + for (v[1] = 0; v[1] < region[1]; ++v[1]) { + std::memcpy( + static_cast<char *>(dst) + dot(dst_pitch, v), + static_cast<const char *>(src) + dot(src_pitch, v), + src_pitch[0] * region[0]); + } + } + }; + } + + /// + /// Hardware copy from \a src_obj to \a dst_obj. + /// + template<typename T, typename S> + std::function<void (event &)> + hard_copy_op(command_queue &q, T dst_obj, const vector_t &dst_orig, + S src_obj, const vector_t &src_orig, const vector_t ®ion) { + return [=, &q](event &) { + dst_obj->resource(q).copy(q, dst_orig, region, + src_obj->resource(q), src_orig); + }; + } +} + +CLOVER_API cl_int +clEnqueueReadBuffer(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, + size_t offset, size_t size, void *ptr, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &mem = obj<buffer>(d_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + vector_t region = { size, 1, 1 }; + vector_t obj_origin = { offset }; + auto obj_pitch = pitch(region, {{ 1 }}); + + validate_common(q, deps); + validate_object(q, ptr, {}, obj_pitch, region); + validate_object(q, mem, obj_origin, obj_pitch, region); + validate_object_access(mem, CL_MEM_HOST_READ_ONLY); + + auto hev = create<hard_event>( + q, CL_COMMAND_READ_BUFFER, deps, + soft_copy_op(q, ptr, {}, obj_pitch, + &mem, obj_origin, obj_pitch, + region)); + + if (blocking) + hev().wait_signalled(); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueWriteBuffer(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, + size_t offset, size_t size, const void *ptr, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &mem = obj<buffer>(d_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + vector_t region = { size, 1, 1 }; + vector_t obj_origin = { offset }; + auto obj_pitch = pitch(region, {{ 1 }}); + + validate_common(q, deps); + validate_object(q, mem, obj_origin, obj_pitch, region); + validate_object(q, ptr, {}, obj_pitch, region); + validate_object_access(mem, CL_MEM_HOST_WRITE_ONLY); + + auto hev = create<hard_event>( + q, CL_COMMAND_WRITE_BUFFER, deps, + soft_copy_op(q, &mem, obj_origin, obj_pitch, + ptr, {}, obj_pitch, + region)); + + if (blocking) + hev().wait_signalled(); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueReadBufferRect(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, + const size_t *p_obj_origin, + const size_t *p_host_origin, + const size_t *p_region, + size_t obj_row_pitch, size_t obj_slice_pitch, + size_t host_row_pitch, size_t host_slice_pitch, + void *ptr, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &mem = obj<buffer>(d_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto region = vector(p_region); + auto obj_origin = vector(p_obj_origin); + auto obj_pitch = pitch(region, {{ 1, obj_row_pitch, obj_slice_pitch }}); + auto host_origin = vector(p_host_origin); + auto host_pitch = pitch(region, {{ 1, host_row_pitch, host_slice_pitch }}); + + validate_common(q, deps); + validate_object(q, ptr, host_origin, host_pitch, region); + validate_object(q, mem, obj_origin, obj_pitch, region); + validate_object_access(mem, CL_MEM_HOST_READ_ONLY); + + auto hev = create<hard_event>( + q, CL_COMMAND_READ_BUFFER_RECT, deps, + soft_copy_op(q, ptr, host_origin, host_pitch, + &mem, obj_origin, obj_pitch, + region)); + + if (blocking) + hev().wait_signalled(); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueWriteBufferRect(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, + const size_t *p_obj_origin, + const size_t *p_host_origin, + const size_t *p_region, + size_t obj_row_pitch, size_t obj_slice_pitch, + size_t host_row_pitch, size_t host_slice_pitch, + const void *ptr, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &mem = obj<buffer>(d_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto region = vector(p_region); + auto obj_origin = vector(p_obj_origin); + auto obj_pitch = pitch(region, {{ 1, obj_row_pitch, obj_slice_pitch }}); + auto host_origin = vector(p_host_origin); + auto host_pitch = pitch(region, {{ 1, host_row_pitch, host_slice_pitch }}); + + validate_common(q, deps); + validate_object(q, mem, obj_origin, obj_pitch, region); + validate_object(q, ptr, host_origin, host_pitch, region); + validate_object_access(mem, CL_MEM_HOST_WRITE_ONLY); + + auto hev = create<hard_event>( + q, CL_COMMAND_WRITE_BUFFER_RECT, deps, + soft_copy_op(q, &mem, obj_origin, obj_pitch, + ptr, host_origin, host_pitch, + region)); + + if (blocking) + hev().wait_signalled(); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueCopyBuffer(cl_command_queue d_q, cl_mem d_src_mem, cl_mem d_dst_mem, + size_t src_offset, size_t dst_offset, size_t size, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &src_mem = obj<buffer>(d_src_mem); + auto &dst_mem = obj<buffer>(d_dst_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + vector_t region = { size, 1, 1 }; + vector_t dst_origin = { dst_offset }; + auto dst_pitch = pitch(region, {{ 1 }}); + vector_t src_origin = { src_offset }; + auto src_pitch = pitch(region, {{ 1 }}); + + validate_common(q, deps); + validate_object(q, dst_mem, dst_origin, dst_pitch, region); + validate_object(q, src_mem, src_origin, src_pitch, region); + validate_copy(q, dst_mem, dst_origin, dst_pitch, + src_mem, src_origin, src_pitch, region); + + auto hev = create<hard_event>( + q, CL_COMMAND_COPY_BUFFER, deps, + hard_copy_op(q, &dst_mem, dst_origin, + &src_mem, src_origin, region)); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueCopyBufferRect(cl_command_queue d_q, cl_mem d_src_mem, + cl_mem d_dst_mem, + const size_t *p_src_origin, const size_t *p_dst_origin, + const size_t *p_region, + size_t src_row_pitch, size_t src_slice_pitch, + size_t dst_row_pitch, size_t dst_slice_pitch, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &src_mem = obj<buffer>(d_src_mem); + auto &dst_mem = obj<buffer>(d_dst_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto region = vector(p_region); + auto dst_origin = vector(p_dst_origin); + auto dst_pitch = pitch(region, {{ 1, dst_row_pitch, dst_slice_pitch }}); + auto src_origin = vector(p_src_origin); + auto src_pitch = pitch(region, {{ 1, src_row_pitch, src_slice_pitch }}); + + validate_common(q, deps); + validate_object(q, dst_mem, dst_origin, dst_pitch, region); + validate_object(q, src_mem, src_origin, src_pitch, region); + validate_copy(q, dst_mem, dst_origin, dst_pitch, + src_mem, src_origin, src_pitch, region); + + auto hev = create<hard_event>( + q, CL_COMMAND_COPY_BUFFER_RECT, deps, + soft_copy_op(q, &dst_mem, dst_origin, dst_pitch, + &src_mem, src_origin, src_pitch, + region)); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueReadImage(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, + const size_t *p_origin, const size_t *p_region, + size_t row_pitch, size_t slice_pitch, void *ptr, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &img = obj<image>(d_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto region = vector(p_region); + auto dst_pitch = pitch(region, {{ img.pixel_size(), + row_pitch, slice_pitch }}); + auto src_origin = vector(p_origin); + auto src_pitch = pitch(region, {{ img.pixel_size(), + img.row_pitch(), img.slice_pitch() }}); + + validate_common(q, deps); + validate_object(q, ptr, {}, dst_pitch, region); + validate_object(q, img, src_origin, region); + validate_object_access(img, CL_MEM_HOST_READ_ONLY); + + auto hev = create<hard_event>( + q, CL_COMMAND_READ_IMAGE, deps, + soft_copy_op(q, ptr, {}, dst_pitch, + &img, src_origin, src_pitch, + region)); + + if (blocking) + hev().wait_signalled(); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueWriteImage(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, + const size_t *p_origin, const size_t *p_region, + size_t row_pitch, size_t slice_pitch, const void *ptr, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &img = obj<image>(d_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto region = vector(p_region); + auto dst_origin = vector(p_origin); + auto dst_pitch = pitch(region, {{ img.pixel_size(), + img.row_pitch(), img.slice_pitch() }}); + auto src_pitch = pitch(region, {{ img.pixel_size(), + row_pitch, slice_pitch }}); + + validate_common(q, deps); + validate_object(q, img, dst_origin, region); + validate_object(q, ptr, {}, src_pitch, region); + validate_object_access(img, CL_MEM_HOST_WRITE_ONLY); + + auto hev = create<hard_event>( + q, CL_COMMAND_WRITE_IMAGE, deps, + soft_copy_op(q, &img, dst_origin, dst_pitch, + ptr, {}, src_pitch, + region)); + + if (blocking) + hev().wait_signalled(); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueCopyImage(cl_command_queue d_q, cl_mem d_src_mem, cl_mem d_dst_mem, + const size_t *p_src_origin, const size_t *p_dst_origin, + const size_t *p_region, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &src_img = obj<image>(d_src_mem); + auto &dst_img = obj<image>(d_dst_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto region = vector(p_region); + auto dst_origin = vector(p_dst_origin); + auto src_origin = vector(p_src_origin); + + validate_common(q, deps); + validate_object(q, dst_img, dst_origin, region); + validate_object(q, src_img, src_origin, region); + validate_copy(q, dst_img, dst_origin, src_img, src_origin, region); + + auto hev = create<hard_event>( + q, CL_COMMAND_COPY_IMAGE, deps, + hard_copy_op(q, &dst_img, dst_origin, + &src_img, src_origin, + region)); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueCopyImageToBuffer(cl_command_queue d_q, + cl_mem d_src_mem, cl_mem d_dst_mem, + const size_t *p_src_origin, const size_t *p_region, + size_t dst_offset, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &src_img = obj<image>(d_src_mem); + auto &dst_mem = obj<buffer>(d_dst_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto region = vector(p_region); + vector_t dst_origin = { dst_offset }; + auto dst_pitch = pitch(region, {{ src_img.pixel_size() }}); + auto src_origin = vector(p_src_origin); + auto src_pitch = pitch(region, {{ src_img.pixel_size(), + src_img.row_pitch(), + src_img.slice_pitch() }}); + + validate_common(q, deps); + validate_object(q, dst_mem, dst_origin, dst_pitch, region); + validate_object(q, src_img, src_origin, region); + + auto hev = create<hard_event>( + q, CL_COMMAND_COPY_IMAGE_TO_BUFFER, deps, + soft_copy_op(q, &dst_mem, dst_origin, dst_pitch, + &src_img, src_origin, src_pitch, + region)); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueCopyBufferToImage(cl_command_queue d_q, + cl_mem d_src_mem, cl_mem d_dst_mem, + size_t src_offset, + const size_t *p_dst_origin, const size_t *p_region, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &src_mem = obj<buffer>(d_src_mem); + auto &dst_img = obj<image>(d_dst_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto region = vector(p_region); + auto dst_origin = vector(p_dst_origin); + auto dst_pitch = pitch(region, {{ dst_img.pixel_size(), + dst_img.row_pitch(), + dst_img.slice_pitch() }}); + vector_t src_origin = { src_offset }; + auto src_pitch = pitch(region, {{ dst_img.pixel_size() }}); + + validate_common(q, deps); + validate_object(q, dst_img, dst_origin, region); + validate_object(q, src_mem, src_origin, src_pitch, region); + + auto hev = create<hard_event>( + q, CL_COMMAND_COPY_BUFFER_TO_IMAGE, deps, + soft_copy_op(q, &dst_img, dst_origin, dst_pitch, + &src_mem, src_origin, src_pitch, + region)); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API void * +clEnqueueMapBuffer(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, + cl_map_flags flags, size_t offset, size_t size, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev, cl_int *r_errcode) try { + auto &q = obj(d_q); + auto &mem = obj<buffer>(d_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + vector_t region = { size, 1, 1 }; + vector_t obj_origin = { offset }; + auto obj_pitch = pitch(region, {{ 1 }}); + + validate_common(q, deps); + validate_object(q, mem, obj_origin, obj_pitch, region); + validate_map_flags(mem, flags); + + void *map = mem.resource(q).add_map(q, flags, blocking, obj_origin, region); + + auto hev = create<hard_event>(q, CL_COMMAND_MAP_BUFFER, deps); + if (blocking) + hev().wait_signalled(); + + ret_object(rd_ev, hev); + ret_error(r_errcode, CL_SUCCESS); + return map; + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API void * +clEnqueueMapImage(cl_command_queue d_q, cl_mem d_mem, cl_bool blocking, + cl_map_flags flags, + const size_t *p_origin, const size_t *p_region, + size_t *row_pitch, size_t *slice_pitch, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev, cl_int *r_errcode) try { + auto &q = obj(d_q); + auto &img = obj<image>(d_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + auto region = vector(p_region); + auto origin = vector(p_origin); + + validate_common(q, deps); + validate_object(q, img, origin, region); + validate_map_flags(img, flags); + + void *map = img.resource(q).add_map(q, flags, blocking, origin, region); + + auto hev = create<hard_event>(q, CL_COMMAND_MAP_IMAGE, deps); + if (blocking) + hev().wait_signalled(); + + ret_object(rd_ev, hev); + ret_error(r_errcode, CL_SUCCESS); + return map; + +} catch (error &e) { + ret_error(r_errcode, e); + return NULL; +} + +CLOVER_API cl_int +clEnqueueUnmapMemObject(cl_command_queue d_q, cl_mem d_mem, void *ptr, + cl_uint num_deps, const cl_event *d_deps, + cl_event *rd_ev) try { + auto &q = obj(d_q); + auto &mem = obj(d_mem); + auto deps = objs<wait_list_tag>(d_deps, num_deps); + + validate_common(q, deps); + + auto hev = create<hard_event>( + q, CL_COMMAND_UNMAP_MEM_OBJECT, deps, + [=, &q, &mem](event &) { + mem.resource(q).del_map(ptr); + }); + + ret_object(rd_ev, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueMigrateMemObjects(cl_command_queue command_queue, + cl_uint num_mem_objects, + const cl_mem *mem_objects, + cl_mem_migration_flags flags, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + CLOVER_NOT_SUPPORTED_UNTIL("1.2"); + return CL_INVALID_VALUE; +} + +cl_int +clover::EnqueueSVMFree(cl_command_queue d_q, + cl_uint num_svm_pointers, + void *svm_pointers[], + void (CL_CALLBACK *pfn_free_func) ( + cl_command_queue queue, cl_uint num_svm_pointers, + void *svm_pointers[], void *user_data), + void *user_data, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd) try { + + if (bool(num_svm_pointers) != bool(svm_pointers)) + return CL_INVALID_VALUE; + + auto &q = obj(d_q); + bool can_emulate = q.device().has_system_svm(); + auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list); + + validate_common(q, deps); + + std::vector<void *> svm_pointers_cpy(svm_pointers, + svm_pointers + num_svm_pointers); + if (!pfn_free_func) { + if (!can_emulate) { + CLOVER_NOT_SUPPORTED_UNTIL("2.0"); + return CL_INVALID_VALUE; + } + pfn_free_func = [](cl_command_queue, cl_uint num_svm_pointers, + void *svm_pointers[], void *) { + for (void *p : range(svm_pointers, num_svm_pointers)) + free(p); + }; + } + + auto hev = create<hard_event>(q, cmd, deps, + [=](clover::event &) mutable { + pfn_free_func(d_q, num_svm_pointers, svm_pointers_cpy.data(), + user_data); + }); + + ret_object(event, hev); + return CL_SUCCESS; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueSVMFree(cl_command_queue d_q, + cl_uint num_svm_pointers, + void *svm_pointers[], + void (CL_CALLBACK *pfn_free_func) ( + cl_command_queue queue, cl_uint num_svm_pointers, + void *svm_pointers[], void *user_data), + void *user_data, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMFree(d_q, num_svm_pointers, svm_pointers, + pfn_free_func, user_data, num_events_in_wait_list, + event_wait_list, event, CL_COMMAND_SVM_FREE); +} + +cl_int +clover::EnqueueSVMMemcpy(cl_command_queue d_q, + cl_bool blocking_copy, + void *dst_ptr, + const void *src_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd) try { + + if (dst_ptr == nullptr || src_ptr == nullptr) + return CL_INVALID_VALUE; + + if (static_cast<size_t>(abs(reinterpret_cast<ptrdiff_t>(dst_ptr) - + reinterpret_cast<ptrdiff_t>(src_ptr))) < size) + return CL_MEM_COPY_OVERLAP; + + auto &q = obj(d_q); + bool can_emulate = q.device().has_system_svm(); + auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list); + + validate_common(q, deps); + + if (can_emulate) { + auto hev = create<hard_event>(q, cmd, deps, + [=](clover::event &) { + memcpy(dst_ptr, src_ptr, size); + }); + + if (blocking_copy) + hev().wait(); + ret_object(event, hev); + return CL_SUCCESS; + } + + CLOVER_NOT_SUPPORTED_UNTIL("2.0"); + return CL_INVALID_VALUE; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueSVMMemcpy(cl_command_queue d_q, + cl_bool blocking_copy, + void *dst_ptr, + const void *src_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMMemcpy(d_q, blocking_copy, dst_ptr, src_ptr, + size, num_events_in_wait_list, event_wait_list, + event, CL_COMMAND_SVM_MEMCPY); +} + +cl_int +clover::EnqueueSVMMemFill(cl_command_queue d_q, + void *svm_ptr, + const void *pattern, + size_t pattern_size, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd) try { + + if (svm_ptr == nullptr || pattern == nullptr || + !util_is_power_of_two_nonzero(pattern_size) || + pattern_size > 128 || + !ptr_is_aligned(svm_ptr, pattern_size) || + size % pattern_size) + return CL_INVALID_VALUE; + + auto &q = obj(d_q); + bool can_emulate = q.device().has_system_svm(); + auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list); + + validate_common(q, deps); + + if (can_emulate) { + auto hev = create<hard_event>(q, cmd, deps, + [=](clover::event &) { + void *ptr = svm_ptr; + for (size_t s = size; s; s -= pattern_size) { + memcpy(ptr, pattern, pattern_size); + ptr = static_cast<uint8_t*>(ptr) + pattern_size; + } + }); + + ret_object(event, hev); + return CL_SUCCESS; + } + + CLOVER_NOT_SUPPORTED_UNTIL("2.0"); + return CL_INVALID_VALUE; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueSVMMemFill(cl_command_queue d_q, + void *svm_ptr, + const void *pattern, + size_t pattern_size, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMMemFill(d_q, svm_ptr, pattern, pattern_size, + size, num_events_in_wait_list, event_wait_list, + event, CL_COMMAND_SVM_MEMFILL); +} + +cl_int +clover::EnqueueSVMMap(cl_command_queue d_q, + cl_bool blocking_map, + cl_map_flags map_flags, + void *svm_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd) try { + + if (svm_ptr == nullptr || size == 0) + return CL_INVALID_VALUE; + + auto &q = obj(d_q); + bool can_emulate = q.device().has_system_svm(); + auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list); + + validate_common(q, deps); + + if (can_emulate) { + auto hev = create<hard_event>(q, cmd, deps, + [](clover::event &) { }); + + ret_object(event, hev); + return CL_SUCCESS; + } + + CLOVER_NOT_SUPPORTED_UNTIL("2.0"); + return CL_INVALID_VALUE; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueSVMMap(cl_command_queue d_q, + cl_bool blocking_map, + cl_map_flags map_flags, + void *svm_ptr, + size_t size, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMMap(d_q, blocking_map, map_flags, svm_ptr, size, + num_events_in_wait_list, event_wait_list, event, + CL_COMMAND_SVM_MAP); +} + +cl_int +clover::EnqueueSVMUnmap(cl_command_queue d_q, + void *svm_ptr, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event, + cl_int cmd) try { + + if (svm_ptr == nullptr) + return CL_INVALID_VALUE; + + auto &q = obj(d_q); + bool can_emulate = q.device().has_system_svm(); + auto deps = objs<wait_list_tag>(event_wait_list, num_events_in_wait_list); + + validate_common(q, deps); + + if (can_emulate) { + auto hev = create<hard_event>(q, cmd, deps, + [](clover::event &) { }); + + ret_object(event, hev); + return CL_SUCCESS; + } + + CLOVER_NOT_SUPPORTED_UNTIL("2.0"); + return CL_INVALID_VALUE; + +} catch (error &e) { + return e.get(); +} + +CLOVER_API cl_int +clEnqueueSVMUnmap(cl_command_queue d_q, + void *svm_ptr, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + + return EnqueueSVMUnmap(d_q, svm_ptr, num_events_in_wait_list, + event_wait_list, event, CL_COMMAND_SVM_UNMAP); +} + +CLOVER_API cl_int +clEnqueueSVMMigrateMem(cl_command_queue d_q, + cl_uint num_svm_pointers, + const void **svm_pointers, + const size_t *sizes, + const cl_mem_migration_flags flags, + cl_uint num_events_in_wait_list, + const cl_event *event_wait_list, + cl_event *event) { + CLOVER_NOT_SUPPORTED_UNTIL("2.1"); + return CL_INVALID_VALUE; +} diff --git a/src/gallium/frontends/clover/api/util.hpp b/src/gallium/frontends/clover/api/util.hpp new file mode 100644 index 00000000000..66bd12597c6 --- /dev/null +++ b/src/gallium/frontends/clover/api/util.hpp @@ -0,0 +1,84 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_API_UTIL_HPP +#define CLOVER_API_UTIL_HPP + +#include <cassert> +#include <iostream> + +#include "core/error.hpp" +#include "core/property.hpp" +#include "util/algorithm.hpp" + +#ifdef HAVE_CLOVER_ICD +#define CLOVER_API +#define CLOVER_ICD_API PUBLIC +#else +#define CLOVER_API PUBLIC +#define CLOVER_ICD_API PUBLIC +#endif + +#define CLOVER_NOT_SUPPORTED_UNTIL(version) \ + do { \ + std::cerr << "CL user error: " << __func__ \ + << "() requires OpenCL version " << (version) \ + << " or greater." << std::endl; \ + } while (0) + +namespace clover { + /// + /// Return an error code in \a p if non-zero. + /// + inline void + ret_error(cl_int *p, const clover::error &e) { + if (p) + *p = e.get(); + } + + /// + /// Return a clover object in \a p if non-zero incrementing the + /// reference count of the object. + /// + template<typename T> + void + ret_object(typename T::descriptor_type **p, + const intrusive_ref<T> &v) { + if (p) { + v().retain(); + *p = desc(v()); + } + } + + /// + /// Return an API object from an intrusive reference to a Clover object, + /// incrementing the reference count of the object. + /// + template<typename T> + typename T::descriptor_type * + ret_object(const intrusive_ref<T> &v) { + v().retain(); + return desc(v()); + } +} + +#endif diff --git a/src/gallium/frontends/clover/core/compiler.hpp b/src/gallium/frontends/clover/core/compiler.hpp new file mode 100644 index 00000000000..6ef30df9b7f --- /dev/null +++ b/src/gallium/frontends/clover/core/compiler.hpp @@ -0,0 +1,68 @@ +// +// Copyright 2019 Red Hat, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_COMPILER_HPP +#define CLOVER_CORE_COMPILER_HPP + +#include "core/device.hpp" +#include "core/module.hpp" +#include "llvm/invocation.hpp" +#include "nir/invocation.hpp" +#include "spirv/invocation.hpp" + +namespace clover { + namespace compiler { + static inline module + compile_program(const std::string &source, const header_map &headers, + const device &dev, const std::string &opts, + std::string &log) { + switch (dev.ir_format()) { +#ifdef HAVE_CLOVER_SPIRV + case PIPE_SHADER_IR_NIR_SERIALIZED: + return llvm::compile_to_spirv(source, headers, dev, opts, log); +#endif + case PIPE_SHADER_IR_NATIVE: + return llvm::compile_program(source, headers, dev, opts, log); + default: + unreachable("device with unsupported IR"); + throw error(CL_INVALID_VALUE); + } + } + + static inline module + link_program(const std::vector<module> &ms, const device &dev, + const std::string &opts, std::string &log) { + switch (dev.ir_format()) { + case PIPE_SHADER_IR_NIR_SERIALIZED: + return nir::spirv_to_nir(spirv::link_program(ms, dev, opts, log), + dev, log); + case PIPE_SHADER_IR_NATIVE: + return llvm::link_program(ms, dev, opts, log); + default: + unreachable("device with unsupported IR"); + throw error(CL_INVALID_VALUE); + } + } + } +} + +#endif diff --git a/src/gallium/frontends/clover/core/context.cpp b/src/gallium/frontends/clover/core/context.cpp new file mode 100644 index 00000000000..c3e20829384 --- /dev/null +++ b/src/gallium/frontends/clover/core/context.cpp @@ -0,0 +1,51 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/context.hpp" + +using namespace clover; + +context::context(const property_list &props, + const ref_vector<device> &devs, + const notify_action ¬ify) : + notify(notify), props(props), devs(devs) { +} + +bool +context::operator==(const context &ctx) const { + return this == &ctx; +} + +bool +context::operator!=(const context &ctx) const { + return this != &ctx; +} + +const context::property_list & +context::properties() const { + return props; +} + +context::device_range +context::devices() const { + return map(evals(), devs); +} diff --git a/src/gallium/frontends/clover/core/context.hpp b/src/gallium/frontends/clover/core/context.hpp new file mode 100644 index 00000000000..7b22ccae78f --- /dev/null +++ b/src/gallium/frontends/clover/core/context.hpp @@ -0,0 +1,67 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_CONTEXT_HPP +#define CLOVER_CORE_CONTEXT_HPP + +#include "core/object.hpp" +#include "core/device.hpp" +#include "core/property.hpp" + +namespace clover { + class context : public ref_counter, public _cl_context { + private: + typedef adaptor_range< + evals, const std::vector<intrusive_ref<device>> & + > device_range; + typedef clover::property_list<cl_context_properties> property_list; + + public: + typedef std::function<void (const char *)> notify_action; + + context(const property_list &props, const ref_vector<device> &devs, + const notify_action ¬ify); + + context(const context &ctx) = delete; + context & + operator=(const context &ctx) = delete; + + bool + operator==(const context &ctx) const; + bool + operator!=(const context &ctx) const; + + const property_list & + properties() const; + + device_range + devices() const; + + const notify_action notify; + + private: + property_list props; + const std::vector<intrusive_ref<device>> devs; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/device.cpp b/src/gallium/frontends/clover/core/device.cpp new file mode 100644 index 00000000000..609885c21bc --- /dev/null +++ b/src/gallium/frontends/clover/core/device.cpp @@ -0,0 +1,337 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include <unistd.h> +#include "core/device.hpp" +#include "core/platform.hpp" +#include "pipe/p_screen.h" +#include "pipe/p_state.h" +#include "util/bitscan.h" +#include "util/u_debug.h" + +using namespace clover; + +namespace { + template<typename T> + std::vector<T> + get_compute_param(pipe_screen *pipe, pipe_shader_ir ir_format, + pipe_compute_cap cap) { + int sz = pipe->get_compute_param(pipe, ir_format, cap, NULL); + std::vector<T> v(sz / sizeof(T)); + + pipe->get_compute_param(pipe, ir_format, cap, &v.front()); + return v; + } +} + +device::device(clover::platform &platform, pipe_loader_device *ldev) : + platform(platform), ldev(ldev) { + pipe = pipe_loader_create_screen(ldev); + if (pipe && pipe->get_param(pipe, PIPE_CAP_COMPUTE)) { + if (supports_ir(PIPE_SHADER_IR_NATIVE)) + return; +#ifdef HAVE_CLOVER_SPIRV + if (supports_ir(PIPE_SHADER_IR_NIR_SERIALIZED)) + return; +#endif + } + if (pipe) + pipe->destroy(pipe); + throw error(CL_INVALID_DEVICE); +} + +device::~device() { + if (pipe) + pipe->destroy(pipe); + if (ldev) + pipe_loader_release(&ldev, 1); +} + +bool +device::operator==(const device &dev) const { + return this == &dev; +} + +cl_device_type +device::type() const { + switch (ldev->type) { + case PIPE_LOADER_DEVICE_SOFTWARE: + return CL_DEVICE_TYPE_CPU; + case PIPE_LOADER_DEVICE_PCI: + case PIPE_LOADER_DEVICE_PLATFORM: + return CL_DEVICE_TYPE_GPU; + default: + unreachable("Unknown device type."); + } +} + +cl_uint +device::vendor_id() const { + switch (ldev->type) { + case PIPE_LOADER_DEVICE_SOFTWARE: + case PIPE_LOADER_DEVICE_PLATFORM: + return 0; + case PIPE_LOADER_DEVICE_PCI: + return ldev->u.pci.vendor_id; + default: + unreachable("Unknown device type."); + } +} + +size_t +device::max_images_read() const { + return PIPE_MAX_SHADER_IMAGES; +} + +size_t +device::max_images_write() const { + return PIPE_MAX_SHADER_IMAGES; +} + +size_t +device::max_image_buffer_size() const { + return pipe->get_param(pipe, PIPE_CAP_MAX_TEXTURE_BUFFER_SIZE); +} + +cl_uint +device::max_image_levels_2d() const { + return util_last_bit(pipe->get_param(pipe, PIPE_CAP_MAX_TEXTURE_2D_SIZE)); +} + +cl_uint +device::max_image_levels_3d() const { + return pipe->get_param(pipe, PIPE_CAP_MAX_TEXTURE_3D_LEVELS); +} + +size_t +device::max_image_array_number() const { + return pipe->get_param(pipe, PIPE_CAP_MAX_TEXTURE_ARRAY_LAYERS); +} + +cl_uint +device::max_samplers() const { + return pipe->get_shader_param(pipe, PIPE_SHADER_COMPUTE, + PIPE_SHADER_CAP_MAX_TEXTURE_SAMPLERS); +} + +cl_ulong +device::max_mem_global() const { + return get_compute_param<uint64_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_MAX_GLOBAL_SIZE)[0]; +} + +cl_ulong +device::max_mem_local() const { + return get_compute_param<uint64_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_MAX_LOCAL_SIZE)[0]; +} + +cl_ulong +device::max_mem_input() const { + return get_compute_param<uint64_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_MAX_INPUT_SIZE)[0]; +} + +cl_ulong +device::max_const_buffer_size() const { + return pipe->get_shader_param(pipe, PIPE_SHADER_COMPUTE, + PIPE_SHADER_CAP_MAX_CONST_BUFFER_SIZE); +} + +cl_uint +device::max_const_buffers() const { + return pipe->get_shader_param(pipe, PIPE_SHADER_COMPUTE, + PIPE_SHADER_CAP_MAX_CONST_BUFFERS); +} + +size_t +device::max_threads_per_block() const { + return get_compute_param<uint64_t>( + pipe, ir_format(), PIPE_COMPUTE_CAP_MAX_THREADS_PER_BLOCK)[0]; +} + +cl_ulong +device::max_mem_alloc_size() const { + return get_compute_param<uint64_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_MAX_MEM_ALLOC_SIZE)[0]; +} + +cl_uint +device::max_clock_frequency() const { + return get_compute_param<uint32_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_MAX_CLOCK_FREQUENCY)[0]; +} + +cl_uint +device::max_compute_units() const { + return get_compute_param<uint32_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_MAX_COMPUTE_UNITS)[0]; +} + +bool +device::image_support() const { + return get_compute_param<uint32_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_IMAGES_SUPPORTED)[0]; +} + +bool +device::has_doubles() const { + return pipe->get_param(pipe, PIPE_CAP_DOUBLES); +} + +bool +device::has_halves() const { + return pipe->get_shader_param(pipe, PIPE_SHADER_COMPUTE, + PIPE_SHADER_CAP_FP16); +} + +bool +device::has_int64_atomics() const { + return pipe->get_shader_param(pipe, PIPE_SHADER_COMPUTE, + PIPE_SHADER_CAP_INT64_ATOMICS); +} + +bool +device::has_unified_memory() const { + return pipe->get_param(pipe, PIPE_CAP_UMA); +} + +cl_uint +device::mem_base_addr_align() const { + return sysconf(_SC_PAGESIZE); +} + +cl_device_svm_capabilities +device::svm_support() const { + // Without CAP_RESOURCE_FROM_USER_MEMORY SVM and CL_MEM_USE_HOST_PTR + // interactions won't work according to spec as clover manages a GPU side + // copy of the host data. + // + // The biggest problem are memory buffers created with CL_MEM_USE_HOST_PTR, + // but the application and/or the kernel updates the memory via SVM and not + // the cl_mem buffer. + // We can't even do proper tracking on what memory might have been accessed + // as the host ptr to the buffer could be within a SVM region, where through + // the CL API there is no reliable way of knowing if a certain cl_mem buffer + // was accessed by a kernel or not and the runtime can't reliably know from + // which side the GPU buffer content needs to be updated. + // + // Another unsolvable scenario is a cl_mem object passed by cl_mem reference + // and SVM pointer into the same kernel at the same time. + if (pipe->get_param(pipe, PIPE_CAP_RESOURCE_FROM_USER_MEMORY) && + pipe->get_param(pipe, PIPE_CAP_SYSTEM_SVM)) + // we can emulate all lower levels if we support fine grain system + return CL_DEVICE_SVM_FINE_GRAIN_SYSTEM | + CL_DEVICE_SVM_COARSE_GRAIN_BUFFER | + CL_DEVICE_SVM_FINE_GRAIN_BUFFER; + return 0; +} + +std::vector<size_t> +device::max_block_size() const { + auto v = get_compute_param<uint64_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_MAX_BLOCK_SIZE); + return { v.begin(), v.end() }; +} + +cl_uint +device::subgroup_size() const { + return get_compute_param<uint32_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_SUBGROUP_SIZE)[0]; +} + +cl_uint +device::address_bits() const { + return get_compute_param<uint32_t>(pipe, ir_format(), + PIPE_COMPUTE_CAP_ADDRESS_BITS)[0]; +} + +std::string +device::device_name() const { + return pipe->get_name(pipe); +} + +std::string +device::vendor_name() const { + return pipe->get_device_vendor(pipe); +} + +enum pipe_shader_ir +device::ir_format() const { + if (supports_ir(PIPE_SHADER_IR_NATIVE)) + return PIPE_SHADER_IR_NATIVE; + + assert(supports_ir(PIPE_SHADER_IR_NIR_SERIALIZED)); + return PIPE_SHADER_IR_NIR_SERIALIZED; +} + +std::string +device::ir_target() const { + std::vector<char> target = get_compute_param<char>( + pipe, ir_format(), PIPE_COMPUTE_CAP_IR_TARGET); + return { target.data() }; +} + +enum pipe_endian +device::endianness() const { + return (enum pipe_endian)pipe->get_param(pipe, PIPE_CAP_ENDIANNESS); +} + +std::string +device::device_version() const { + static const std::string device_version = + debug_get_option("CLOVER_DEVICE_VERSION_OVERRIDE", "1.1"); + return device_version; +} + +std::string +device::device_clc_version() const { + static const std::string device_clc_version = + debug_get_option("CLOVER_DEVICE_CLC_VERSION_OVERRIDE", "1.1"); + return device_clc_version; +} + +bool +device::supports_ir(enum pipe_shader_ir ir) const { + return pipe->get_shader_param(pipe, PIPE_SHADER_COMPUTE, + PIPE_SHADER_CAP_SUPPORTED_IRS) & (1 << ir); +} + +std::string +device::supported_extensions() const { + return + "cl_khr_byte_addressable_store" + " cl_khr_global_int32_base_atomics" + " cl_khr_global_int32_extended_atomics" + " cl_khr_local_int32_base_atomics" + " cl_khr_local_int32_extended_atomics" + + std::string(has_int64_atomics() ? " cl_khr_int64_base_atomics" : "") + + std::string(has_int64_atomics() ? " cl_khr_int64_extended_atomics" : "") + + std::string(has_doubles() ? " cl_khr_fp64" : "") + + std::string(has_halves() ? " cl_khr_fp16" : "") + + std::string(svm_support() ? " cl_arm_shared_virtual_memory" : ""); +} + +const void * +device::get_compiler_options(enum pipe_shader_ir ir) const { + return pipe->get_compiler_options(pipe, ir, PIPE_SHADER_COMPUTE); +} diff --git a/src/gallium/frontends/clover/core/device.hpp b/src/gallium/frontends/clover/core/device.hpp new file mode 100644 index 00000000000..597f9489b2c --- /dev/null +++ b/src/gallium/frontends/clover/core/device.hpp @@ -0,0 +1,109 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_DEVICE_HPP +#define CLOVER_CORE_DEVICE_HPP + +#include <set> +#include <vector> + +#include "core/object.hpp" +#include "core/format.hpp" +#include "pipe-loader/pipe_loader.h" + +namespace clover { + class platform; + class root_resource; + class hard_event; + + class device : public ref_counter, public _cl_device_id { + public: + device(clover::platform &platform, pipe_loader_device *ldev); + ~device(); + + device(const device &dev) = delete; + device & + operator=(const device &dev) = delete; + + bool + operator==(const device &dev) const; + + cl_device_type type() const; + cl_uint vendor_id() const; + size_t max_images_read() const; + size_t max_images_write() const; + size_t max_image_buffer_size() const; + cl_uint max_image_levels_2d() const; + cl_uint max_image_levels_3d() const; + size_t max_image_array_number() const; + cl_uint max_samplers() const; + cl_ulong max_mem_global() const; + cl_ulong max_mem_local() const; + cl_ulong max_mem_input() const; + cl_ulong max_const_buffer_size() const; + cl_uint max_const_buffers() const; + size_t max_threads_per_block() const; + cl_ulong max_mem_alloc_size() const; + cl_uint max_clock_frequency() const; + cl_uint max_compute_units() const; + bool image_support() const; + bool has_doubles() const; + bool has_halves() const; + bool has_int64_atomics() const; + bool has_unified_memory() const; + cl_uint mem_base_addr_align() const; + cl_device_svm_capabilities svm_support() const; + + std::vector<size_t> max_block_size() const; + cl_uint subgroup_size() const; + cl_uint address_bits() const; + std::string device_name() const; + std::string vendor_name() const; + std::string device_version() const; + std::string device_clc_version() const; + enum pipe_shader_ir ir_format() const; + std::string ir_target() const; + enum pipe_endian endianness() const; + bool supports_ir(enum pipe_shader_ir ir) const; + std::string supported_extensions() const; + + friend class command_queue; + friend class root_resource; + friend class hard_event; + friend std::set<cl_image_format> + supported_formats(const context &, cl_mem_object_type); + const void *get_compiler_options(enum pipe_shader_ir ir) const; + + clover::platform &platform; + + inline bool + has_system_svm() const { + return svm_support() & CL_DEVICE_SVM_FINE_GRAIN_SYSTEM; + } + + private: + pipe_screen *pipe; + pipe_loader_device *ldev; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/error.hpp b/src/gallium/frontends/clover/core/error.hpp new file mode 100644 index 00000000000..0490c19a276 --- /dev/null +++ b/src/gallium/frontends/clover/core/error.hpp @@ -0,0 +1,202 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_ERROR_HPP +#define CLOVER_CORE_ERROR_HPP + +#include "CL/cl.h" + +#include <stdexcept> +#include <string> + +namespace clover { + class command_queue; + class context; + class device; + class event; + class hard_event; + class soft_event; + class kernel; + class memory_obj; + class buffer; + class root_buffer; + class sub_buffer; + class image; + class image2d; + class image3d; + class platform; + class program; + class sampler; + + /// + /// Class that represents an error that can be converted to an + /// OpenCL status code. + /// + class error : public std::runtime_error { + public: + error(cl_int code, std::string what = "") : + std::runtime_error(what), code(code) { + } + + cl_int get() const { + return code; + } + + protected: + cl_int code; + }; + + class invalid_build_options_error : public error { + public: + invalid_build_options_error(const std::string &what = "") : + error(CL_INVALID_BUILD_OPTIONS, what) {} + }; + + class build_error : public error { + public: + build_error(const std::string &what = "") : + error(CL_BUILD_PROGRAM_FAILURE, what) {} + }; + + template<typename O> + class invalid_object_error; + + template<> + class invalid_object_error<command_queue> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_COMMAND_QUEUE, what) {} + }; + + template<> + class invalid_object_error<context> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_CONTEXT, what) {} + }; + + template<> + class invalid_object_error<device> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_DEVICE, what) {} + }; + + template<> + class invalid_object_error<event> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_EVENT, what) {} + }; + + template<> + class invalid_object_error<soft_event> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_EVENT, what) {} + }; + + template<> + class invalid_object_error<kernel> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_KERNEL, what) {} + }; + + template<> + class invalid_object_error<memory_obj> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_MEM_OBJECT, what) {} + }; + + template<> + class invalid_object_error<buffer> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_MEM_OBJECT, what) {} + }; + + template<> + class invalid_object_error<root_buffer> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_MEM_OBJECT, what) {} + }; + + template<> + class invalid_object_error<sub_buffer> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_MEM_OBJECT, what) {} + }; + + template<> + class invalid_object_error<image> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_MEM_OBJECT, what) {} + }; + + template<> + class invalid_object_error<image2d> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_MEM_OBJECT, what) {} + }; + + template<> + class invalid_object_error<image3d> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_MEM_OBJECT, what) {} + }; + + template<> + class invalid_object_error<platform> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_PLATFORM, what) {} + }; + + template<> + class invalid_object_error<program> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_PROGRAM, what) {} + }; + + template<> + class invalid_object_error<sampler> : public error { + public: + invalid_object_error(std::string what = "") : + error(CL_INVALID_SAMPLER, what) {} + }; + + class invalid_wait_list_error : public error { + public: + invalid_wait_list_error(std::string what = "") : + error(CL_INVALID_EVENT_WAIT_LIST, what) {} + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/event.cpp b/src/gallium/frontends/clover/core/event.cpp new file mode 100644 index 00000000000..3d313ce8969 --- /dev/null +++ b/src/gallium/frontends/clover/core/event.cpp @@ -0,0 +1,267 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/event.hpp" +#include "pipe/p_screen.h" + +using namespace clover; + +event::event(clover::context &ctx, const ref_vector<event> &deps, + action action_ok, action action_fail) : + context(ctx), _wait_count(1), _status(0), + action_ok(action_ok), action_fail(action_fail) { + for (auto &ev : deps) + ev.chain(*this); +} + +event::~event() { +} + +std::vector<intrusive_ref<event>> +event::trigger_self() { + std::lock_guard<std::mutex> lock(mutex); + std::vector<intrusive_ref<event>> evs; + + if (_wait_count && !--_wait_count) + std::swap(_chain, evs); + + cv.notify_all(); + return evs; +} + +void +event::trigger() try { + if (wait_count() == 1) + action_ok(*this); + + for (event &ev : trigger_self()) + ev.trigger(); +} catch (error &e) { + abort(e.get()); +} + +std::vector<intrusive_ref<event>> +event::abort_self(cl_int status) { + std::lock_guard<std::mutex> lock(mutex); + std::vector<intrusive_ref<event>> evs; + + _status = status; + _wait_count = 0; + std::swap(_chain, evs); + + cv.notify_all(); + return evs; +} + +void +event::abort(cl_int status) { + action_fail(*this); + + for (event &ev : abort_self(status)) + ev.abort(status); +} + +unsigned +event::wait_count() const { + std::lock_guard<std::mutex> lock(mutex); + return _wait_count; +} + +bool +event::signalled() const { + return !wait_count(); +} + +cl_int +event::status() const { + std::lock_guard<std::mutex> lock(mutex); + return _status; +} + +void +event::chain(event &ev) { + std::unique_lock<std::mutex> lock(mutex, std::defer_lock); + std::unique_lock<std::mutex> lock_ev(ev.mutex, std::defer_lock); + std::lock(lock, lock_ev); + + if (_wait_count) { + ev._wait_count++; + _chain.push_back(ev); + } + ev.deps.push_back(*this); +} + +void +event::wait_signalled() const { + std::unique_lock<std::mutex> lock(mutex); + cv.wait(lock, [=]{ return !_wait_count; }); +} + +void +event::wait() const { + for (event &ev : deps) + ev.wait(); + + wait_signalled(); +} + +hard_event::hard_event(command_queue &q, cl_command_type command, + const ref_vector<event> &deps, action action) : + event(q.context(), deps, profile(q, action), [](event &ev){}), + _queue(q), _command(command), _fence(NULL) { + if (q.profiling_enabled()) + _time_queued = timestamp::current(q); + + q.sequence(*this); + trigger(); +} + +hard_event::~hard_event() { + pipe_screen *screen = queue()->device().pipe; + screen->fence_reference(screen, &_fence, NULL); +} + +cl_int +hard_event::status() const { + pipe_screen *screen = queue()->device().pipe; + + if (event::status() < 0) + return event::status(); + + else if (!_fence) + return CL_QUEUED; + + else if (!screen->fence_finish(screen, NULL, _fence, 0)) + return CL_SUBMITTED; + + else + return CL_COMPLETE; +} + +command_queue * +hard_event::queue() const { + return &_queue(); +} + +cl_command_type +hard_event::command() const { + return _command; +} + +void +hard_event::wait() const { + pipe_screen *screen = queue()->device().pipe; + + event::wait(); + + if (status() == CL_QUEUED) + queue()->flush(); + + if (!_fence || + !screen->fence_finish(screen, NULL, _fence, PIPE_TIMEOUT_INFINITE)) + throw error(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST); +} + +const lazy<cl_ulong> & +hard_event::time_queued() const { + return _time_queued; +} + +const lazy<cl_ulong> & +hard_event::time_submit() const { + return _time_submit; +} + +const lazy<cl_ulong> & +hard_event::time_start() const { + return _time_start; +} + +const lazy<cl_ulong> & +hard_event::time_end() const { + return _time_end; +} + +void +hard_event::fence(pipe_fence_handle *fence) { + pipe_screen *screen = queue()->device().pipe; + screen->fence_reference(screen, &_fence, fence); +} + +event::action +hard_event::profile(command_queue &q, const action &action) const { + if (q.profiling_enabled()) { + return [&q, action] (event &ev) { + auto &hev = static_cast<hard_event &>(ev); + + hev._time_submit = timestamp::current(q); + hev._time_start = timestamp::query(q); + + action(ev); + + hev._time_end = timestamp::query(q); + }; + + } else { + return action; + } +} + +soft_event::soft_event(clover::context &ctx, const ref_vector<event> &deps, + bool _trigger, action action) : + event(ctx, deps, action, action) { + if (_trigger) + trigger(); +} + +cl_int +soft_event::status() const { + if (event::status() < 0) + return event::status(); + + else if (!signalled() || + any_of([](const event &ev) { + return ev.status() != CL_COMPLETE; + }, deps)) + return CL_SUBMITTED; + + else + return CL_COMPLETE; +} + +command_queue * +soft_event::queue() const { + return NULL; +} + +cl_command_type +soft_event::command() const { + return CL_COMMAND_USER; +} + +void +soft_event::wait() const { + event::wait(); + + if (status() != CL_COMPLETE) + throw error(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST); +} diff --git a/src/gallium/frontends/clover/core/event.hpp b/src/gallium/frontends/clover/core/event.hpp new file mode 100644 index 00000000000..03c97bcf4da --- /dev/null +++ b/src/gallium/frontends/clover/core/event.hpp @@ -0,0 +1,164 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_EVENT_HPP +#define CLOVER_CORE_EVENT_HPP + +#include <condition_variable> +#include <functional> + +#include "core/object.hpp" +#include "core/queue.hpp" +#include "core/timestamp.hpp" +#include "util/lazy.hpp" + +namespace clover { + /// + /// Class that represents a task that might be executed + /// asynchronously at some point in the future. + /// + /// An event consists of a list of dependencies, a boolean + /// signalled() flag, and an associated task. An event is + /// considered signalled as soon as all its dependencies (if any) + /// are signalled as well, and the trigger() method is called; at + /// that point the associated task will be started through the + /// specified \a action_ok. If the abort() method is called + /// instead, the specified \a action_fail is executed and the + /// associated task will never be started. Dependent events will + /// be aborted recursively. + /// + /// The execution status of the associated task can be queried + /// using the status() method, and it can be waited for completion + /// using the wait() method. + /// + class event : public ref_counter, public _cl_event { + public: + typedef std::function<void (event &)> action; + + event(clover::context &ctx, const ref_vector<event> &deps, + action action_ok, action action_fail); + virtual ~event(); + + event(const event &ev) = delete; + event & + operator=(const event &ev) = delete; + + void trigger(); + void abort(cl_int status); + bool signalled() const; + + virtual cl_int status() const; + virtual command_queue *queue() const = 0; + virtual cl_command_type command() const = 0; + void wait_signalled() const; + virtual void wait() const; + + virtual struct pipe_fence_handle *fence() const { + return NULL; + } + + const intrusive_ref<clover::context> context; + + protected: + void chain(event &ev); + + std::vector<intrusive_ref<event>> deps; + + private: + std::vector<intrusive_ref<event>> trigger_self(); + std::vector<intrusive_ref<event>> abort_self(cl_int status); + unsigned wait_count() const; + + unsigned _wait_count; + cl_int _status; + action action_ok; + action action_fail; + std::vector<intrusive_ref<event>> _chain; + mutable std::condition_variable cv; + mutable std::mutex mutex; + }; + + /// + /// Class that represents a task executed by a command queue. + /// + /// Similar to a normal clover::event. In addition it's associated + /// with a given command queue \a q and a given OpenCL \a command. + /// hard_event instances created for the same queue are implicitly + /// ordered with respect to each other, and they are implicitly + /// triggered on construction. + /// + /// A hard_event is considered complete when the associated + /// hardware task finishes execution. + /// + class hard_event : public event { + public: + hard_event(command_queue &q, cl_command_type command, + const ref_vector<event> &deps, + action action = [](event &){}); + ~hard_event(); + + virtual cl_int status() const; + virtual command_queue *queue() const; + virtual cl_command_type command() const; + virtual void wait() const; + + const lazy<cl_ulong> &time_queued() const; + const lazy<cl_ulong> &time_submit() const; + const lazy<cl_ulong> &time_start() const; + const lazy<cl_ulong> &time_end() const; + + friend class command_queue; + + virtual struct pipe_fence_handle *fence() const { + return _fence; + } + + private: + virtual void fence(pipe_fence_handle *fence); + action profile(command_queue &q, const action &action) const; + + const intrusive_ref<command_queue> _queue; + cl_command_type _command; + pipe_fence_handle *_fence; + lazy<cl_ulong> _time_queued, _time_submit, _time_start, _time_end; + }; + + /// + /// Class that represents a software event. + /// + /// A soft_event is not associated with any specific hardware task + /// or command queue. It's considered complete as soon as all its + /// dependencies finish execution. + /// + class soft_event : public event { + public: + soft_event(clover::context &ctx, const ref_vector<event> &deps, + bool trigger, action action = [](event &){}); + + virtual cl_int status() const; + virtual command_queue *queue() const; + virtual cl_command_type command() const; + virtual void wait() const; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/format.cpp b/src/gallium/frontends/clover/core/format.cpp new file mode 100644 index 00000000000..dee1872c829 --- /dev/null +++ b/src/gallium/frontends/clover/core/format.cpp @@ -0,0 +1,162 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/format.hpp" +#include "core/memory.hpp" +#include "pipe/p_screen.h" +#include "pipe/p_context.h" + +namespace clover { + static const std::map<cl_image_format, pipe_format> formats { + { { CL_BGRA, CL_UNORM_INT8 }, PIPE_FORMAT_B8G8R8A8_UNORM }, + { { CL_ARGB, CL_UNORM_INT8 }, PIPE_FORMAT_A8R8G8B8_UNORM }, + { { CL_RGB, CL_UNORM_SHORT_565 }, PIPE_FORMAT_B5G6R5_UNORM }, + { { CL_LUMINANCE, CL_UNORM_INT8 }, PIPE_FORMAT_L8_UNORM }, + { { CL_A, CL_UNORM_INT8 }, PIPE_FORMAT_A8_UNORM }, + { { CL_INTENSITY, CL_UNORM_INT8 }, PIPE_FORMAT_I8_UNORM }, + { { CL_LUMINANCE, CL_UNORM_INT16 }, PIPE_FORMAT_L16_UNORM }, + { { CL_R, CL_FLOAT }, PIPE_FORMAT_R32_FLOAT }, + { { CL_RG, CL_FLOAT }, PIPE_FORMAT_R32G32_FLOAT }, + { { CL_RGB, CL_FLOAT }, PIPE_FORMAT_R32G32B32_FLOAT }, + { { CL_RGBA, CL_FLOAT }, PIPE_FORMAT_R32G32B32A32_FLOAT }, + { { CL_R, CL_UNORM_INT16 }, PIPE_FORMAT_R16_UNORM }, + { { CL_RG, CL_UNORM_INT16 }, PIPE_FORMAT_R16G16_UNORM }, + { { CL_RGB, CL_UNORM_INT16 }, PIPE_FORMAT_R16G16B16_UNORM }, + { { CL_RGBA, CL_UNORM_INT16 }, PIPE_FORMAT_R16G16B16A16_UNORM }, + { { CL_R, CL_SNORM_INT16 }, PIPE_FORMAT_R16_SNORM }, + { { CL_RG, CL_SNORM_INT16 }, PIPE_FORMAT_R16G16_SNORM }, + { { CL_RGB, CL_SNORM_INT16 }, PIPE_FORMAT_R16G16B16_SNORM }, + { { CL_RGBA, CL_SNORM_INT16 }, PIPE_FORMAT_R16G16B16A16_SNORM }, + { { CL_R, CL_UNORM_INT8 }, PIPE_FORMAT_R8_UNORM }, + { { CL_RG, CL_UNORM_INT8 }, PIPE_FORMAT_R8G8_UNORM }, + { { CL_RGB, CL_UNORM_INT8 }, PIPE_FORMAT_R8G8B8_UNORM }, + { { CL_RGBA, CL_UNORM_INT8 }, PIPE_FORMAT_R8G8B8A8_UNORM }, + { { CL_R, CL_SNORM_INT8 }, PIPE_FORMAT_R8_SNORM }, + { { CL_RG, CL_SNORM_INT8 }, PIPE_FORMAT_R8G8_SNORM }, + { { CL_RGB, CL_SNORM_INT8 }, PIPE_FORMAT_R8G8B8_SNORM }, + { { CL_RGBA, CL_SNORM_INT8 }, PIPE_FORMAT_R8G8B8A8_SNORM }, + { { CL_R, CL_HALF_FLOAT }, PIPE_FORMAT_R16_FLOAT }, + { { CL_RG, CL_HALF_FLOAT }, PIPE_FORMAT_R16G16_FLOAT }, + { { CL_RGB, CL_HALF_FLOAT }, PIPE_FORMAT_R16G16B16_FLOAT }, + { { CL_RGBA, CL_HALF_FLOAT }, PIPE_FORMAT_R16G16B16A16_FLOAT }, + { { CL_RGBx, CL_UNORM_SHORT_555 }, PIPE_FORMAT_B5G5R5X1_UNORM }, + { { CL_RGBx, CL_UNORM_INT8 }, PIPE_FORMAT_R8G8B8X8_UNORM }, + { { CL_A, CL_UNORM_INT16 }, PIPE_FORMAT_A16_UNORM }, + { { CL_INTENSITY, CL_UNORM_INT16 }, PIPE_FORMAT_I16_UNORM }, + { { CL_LUMINANCE, CL_SNORM_INT8 }, PIPE_FORMAT_L8_SNORM }, + { { CL_INTENSITY, CL_SNORM_INT8 }, PIPE_FORMAT_I8_SNORM }, + { { CL_A, CL_SNORM_INT16 }, PIPE_FORMAT_A16_SNORM }, + { { CL_LUMINANCE, CL_SNORM_INT16 }, PIPE_FORMAT_L16_SNORM }, + { { CL_INTENSITY, CL_SNORM_INT16 }, PIPE_FORMAT_I16_SNORM }, + { { CL_A, CL_HALF_FLOAT }, PIPE_FORMAT_A16_FLOAT }, + { { CL_LUMINANCE, CL_HALF_FLOAT }, PIPE_FORMAT_L16_FLOAT }, + { { CL_INTENSITY, CL_HALF_FLOAT }, PIPE_FORMAT_I16_FLOAT }, + { { CL_A, CL_FLOAT }, PIPE_FORMAT_A32_FLOAT }, + { { CL_LUMINANCE, CL_FLOAT }, PIPE_FORMAT_L32_FLOAT }, + { { CL_INTENSITY, CL_FLOAT }, PIPE_FORMAT_I32_FLOAT }, + { { CL_RA, CL_UNORM_INT8 }, PIPE_FORMAT_R8A8_UNORM }, + { { CL_R, CL_UNSIGNED_INT8 }, PIPE_FORMAT_R8_UINT }, + { { CL_RG, CL_UNSIGNED_INT8 }, PIPE_FORMAT_R8G8_UINT }, + { { CL_RGB, CL_UNSIGNED_INT8 }, PIPE_FORMAT_R8G8B8_UINT }, + { { CL_RGBA, CL_UNSIGNED_INT8 }, PIPE_FORMAT_R8G8B8A8_UINT }, + { { CL_R, CL_SIGNED_INT8 }, PIPE_FORMAT_R8_SINT }, + { { CL_RG, CL_SIGNED_INT8 }, PIPE_FORMAT_R8G8_SINT }, + { { CL_RGB, CL_SIGNED_INT8 }, PIPE_FORMAT_R8G8B8_SINT }, + { { CL_RGBA, CL_SIGNED_INT8 }, PIPE_FORMAT_R8G8B8A8_SINT }, + { { CL_R, CL_UNSIGNED_INT16 }, PIPE_FORMAT_R16_UINT }, + { { CL_RG, CL_UNSIGNED_INT16 }, PIPE_FORMAT_R16G16_UINT }, + { { CL_RGB, CL_UNSIGNED_INT16 }, PIPE_FORMAT_R16G16B16_UINT }, + { { CL_RGBA, CL_UNSIGNED_INT16 }, PIPE_FORMAT_R16G16B16A16_UINT }, + { { CL_R, CL_SIGNED_INT16 }, PIPE_FORMAT_R16_SINT }, + { { CL_RG, CL_SIGNED_INT16 }, PIPE_FORMAT_R16G16_SINT }, + { { CL_RGB, CL_SIGNED_INT16 }, PIPE_FORMAT_R16G16B16_SINT }, + { { CL_RGBA, CL_SIGNED_INT16 }, PIPE_FORMAT_R16G16B16A16_SINT }, + { { CL_R, CL_UNSIGNED_INT32 }, PIPE_FORMAT_R32_UINT }, + { { CL_RG, CL_UNSIGNED_INT32 }, PIPE_FORMAT_R32G32_UINT }, + { { CL_RGB, CL_UNSIGNED_INT32 }, PIPE_FORMAT_R32G32B32_UINT }, + { { CL_RGBA, CL_UNSIGNED_INT32 }, PIPE_FORMAT_R32G32B32A32_UINT }, + { { CL_R, CL_SIGNED_INT32 }, PIPE_FORMAT_R32_SINT }, + { { CL_RG, CL_SIGNED_INT32 }, PIPE_FORMAT_R32G32_SINT }, + { { CL_RGB, CL_SIGNED_INT32 }, PIPE_FORMAT_R32G32B32_SINT }, + { { CL_RGBA, CL_SIGNED_INT32 }, PIPE_FORMAT_R32G32B32A32_SINT }, + { { CL_A, CL_UNSIGNED_INT8 }, PIPE_FORMAT_A8_UINT }, + { { CL_INTENSITY, CL_UNSIGNED_INT8 }, PIPE_FORMAT_I8_UINT }, + { { CL_LUMINANCE, CL_UNSIGNED_INT8 }, PIPE_FORMAT_L8_UINT }, + { { CL_A, CL_SIGNED_INT8 }, PIPE_FORMAT_A8_SINT }, + { { CL_INTENSITY, CL_SIGNED_INT8 }, PIPE_FORMAT_I8_SINT }, + { { CL_LUMINANCE, CL_SIGNED_INT8 }, PIPE_FORMAT_L8_SINT }, + { { CL_A, CL_UNSIGNED_INT16 }, PIPE_FORMAT_A16_UINT }, + { { CL_INTENSITY, CL_UNSIGNED_INT16 }, PIPE_FORMAT_I16_UINT }, + { { CL_LUMINANCE, CL_UNSIGNED_INT16 }, PIPE_FORMAT_L16_UINT }, + { { CL_A, CL_SIGNED_INT16 }, PIPE_FORMAT_A16_SINT }, + { { CL_INTENSITY, CL_SIGNED_INT16 }, PIPE_FORMAT_I16_SINT }, + { { CL_LUMINANCE, CL_SIGNED_INT16 }, PIPE_FORMAT_L16_SINT }, + { { CL_A, CL_UNSIGNED_INT32 }, PIPE_FORMAT_A32_UINT }, + { { CL_INTENSITY, CL_UNSIGNED_INT32 }, PIPE_FORMAT_I32_UINT }, + { { CL_LUMINANCE, CL_UNSIGNED_INT32 }, PIPE_FORMAT_L32_UINT }, + { { CL_A, CL_SIGNED_INT32 }, PIPE_FORMAT_A32_SINT }, + { { CL_INTENSITY, CL_SIGNED_INT32 }, PIPE_FORMAT_I32_SINT }, + { { CL_LUMINANCE, CL_SIGNED_INT32 }, PIPE_FORMAT_L32_SINT } + }; + + pipe_texture_target + translate_target(cl_mem_object_type type) { + switch (type) { + case CL_MEM_OBJECT_BUFFER: + return PIPE_BUFFER; + case CL_MEM_OBJECT_IMAGE2D: + return PIPE_TEXTURE_2D; + case CL_MEM_OBJECT_IMAGE3D: + return PIPE_TEXTURE_3D; + default: + throw error(CL_INVALID_VALUE); + } + } + + pipe_format + translate_format(const cl_image_format &format) { + auto it = formats.find(format); + + if (it == formats.end()) + throw error(CL_IMAGE_FORMAT_NOT_SUPPORTED); + + return it->second; + } + + std::set<cl_image_format> + supported_formats(const context &ctx, cl_mem_object_type type) { + std::set<cl_image_format> s; + pipe_texture_target target = translate_target(type); + unsigned bindings = (PIPE_BIND_SAMPLER_VIEW | + PIPE_BIND_COMPUTE_RESOURCE); + + for (auto f : formats) { + if (all_of([=](const device &dev) { + return dev.pipe->is_format_supported( + dev.pipe, f.second, target, 1, 1, bindings); + }, ctx.devices())) + s.insert(f.first); + } + + return s; + } +} diff --git a/src/gallium/frontends/clover/core/format.hpp b/src/gallium/frontends/clover/core/format.hpp new file mode 100644 index 00000000000..a8b7053c5dc --- /dev/null +++ b/src/gallium/frontends/clover/core/format.hpp @@ -0,0 +1,62 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_FORMAT_HPP +#define CLOVER_CORE_FORMAT_HPP + +#include <set> + +#include "core/object.hpp" +#include "pipe/p_defines.h" +#include "pipe/p_format.h" + +namespace clover { + pipe_texture_target translate_target(cl_mem_object_type type); + pipe_format translate_format(const cl_image_format &format); + + /// + /// Return all the image formats supported by a given context for + /// the given memory object type. + /// + std::set<cl_image_format> supported_formats(const context &ctx, + cl_mem_object_type type); +} + +static inline bool +operator<(const cl_image_format &a, const cl_image_format &b) { + return (a.image_channel_order != b.image_channel_order ? + a.image_channel_order < b.image_channel_order : + a.image_channel_data_type < b.image_channel_data_type); +} + +static inline bool +operator==(const cl_image_format &a, const cl_image_format &b) { + return (a.image_channel_order == b.image_channel_order && + a.image_channel_data_type == b.image_channel_data_type); +} + +static inline bool +operator!=(const cl_image_format &a, const cl_image_format &b) { + return !(a == b); +} + +#endif diff --git a/src/gallium/frontends/clover/core/kernel.cpp b/src/gallium/frontends/clover/core/kernel.cpp new file mode 100644 index 00000000000..7d839767aa0 --- /dev/null +++ b/src/gallium/frontends/clover/core/kernel.cpp @@ -0,0 +1,610 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/kernel.hpp" +#include "core/resource.hpp" +#include "util/factor.hpp" +#include "util/u_math.h" +#include "pipe/p_context.h" + +using namespace clover; + +kernel::kernel(clover::program &prog, const std::string &name, + const std::vector<module::argument> &margs) : + program(prog), _name(name), exec(*this), + program_ref(prog._kernel_ref_counter) { + for (auto &marg : margs) { + if (marg.semantic == module::argument::general) + _args.emplace_back(argument::create(marg)); + } +} + +template<typename V> +static inline std::vector<uint> +pad_vector(command_queue &q, const V &v, uint x) { + std::vector<uint> w { v.begin(), v.end() }; + w.resize(q.device().max_block_size().size(), x); + return w; +} + +void +kernel::launch(command_queue &q, + const std::vector<size_t> &grid_offset, + const std::vector<size_t> &grid_size, + const std::vector<size_t> &block_size) { + const auto m = program().build(q.device()).binary; + const auto reduced_grid_size = + map(divides(), grid_size, block_size); + void *st = exec.bind(&q, grid_offset); + struct pipe_grid_info info = {}; + + // The handles are created during exec_context::bind(), so we need make + // sure to call exec_context::bind() before retrieving them. + std::vector<uint32_t *> g_handles = map([&](size_t h) { + return (uint32_t *)&exec.input[h]; + }, exec.g_handles); + + q.pipe->bind_compute_state(q.pipe, st); + q.pipe->bind_sampler_states(q.pipe, PIPE_SHADER_COMPUTE, + 0, exec.samplers.size(), + exec.samplers.data()); + + q.pipe->set_sampler_views(q.pipe, PIPE_SHADER_COMPUTE, 0, + exec.sviews.size(), exec.sviews.data()); + q.pipe->set_compute_resources(q.pipe, 0, exec.resources.size(), + exec.resources.data()); + q.pipe->set_global_binding(q.pipe, 0, exec.g_buffers.size(), + exec.g_buffers.data(), g_handles.data()); + + // Fill information for the launch_grid() call. + info.work_dim = grid_size.size(); + copy(pad_vector(q, block_size, 1), info.block); + copy(pad_vector(q, reduced_grid_size, 1), info.grid); + info.pc = find(name_equals(_name), m.syms).offset; + info.input = exec.input.data(); + + q.pipe->launch_grid(q.pipe, &info); + + q.pipe->set_global_binding(q.pipe, 0, exec.g_buffers.size(), NULL, NULL); + q.pipe->set_compute_resources(q.pipe, 0, exec.resources.size(), NULL); + q.pipe->set_sampler_views(q.pipe, PIPE_SHADER_COMPUTE, 0, + exec.sviews.size(), NULL); + q.pipe->bind_sampler_states(q.pipe, PIPE_SHADER_COMPUTE, 0, + exec.samplers.size(), NULL); + + q.pipe->memory_barrier(q.pipe, PIPE_BARRIER_GLOBAL_BUFFER); + exec.unbind(); +} + +size_t +kernel::mem_local() const { + size_t sz = 0; + + for (auto &arg : args()) { + if (dynamic_cast<local_argument *>(&arg)) + sz += arg.storage(); + } + + return sz; +} + +size_t +kernel::mem_private() const { + return 0; +} + +const std::string & +kernel::name() const { + return _name; +} + +std::vector<size_t> +kernel::optimal_block_size(const command_queue &q, + const std::vector<size_t> &grid_size) const { + return factor::find_grid_optimal_factor<size_t>( + q.device().max_threads_per_block(), q.device().max_block_size(), + grid_size); +} + +std::vector<size_t> +kernel::required_block_size() const { + return { 0, 0, 0 }; +} + +kernel::argument_range +kernel::args() { + return map(derefs(), _args); +} + +kernel::const_argument_range +kernel::args() const { + return map(derefs(), _args); +} + +const module & +kernel::module(const command_queue &q) const { + return program().build(q.device()).binary; +} + +kernel::exec_context::exec_context(kernel &kern) : + kern(kern), q(NULL), mem_local(0), st(NULL), cs() { +} + +kernel::exec_context::~exec_context() { + if (st) + q->pipe->delete_compute_state(q->pipe, st); +} + +void * +kernel::exec_context::bind(intrusive_ptr<command_queue> _q, + const std::vector<size_t> &grid_offset) { + std::swap(q, _q); + + // Bind kernel arguments. + auto &m = kern.program().build(q->device()).binary; + auto msym = find(name_equals(kern.name()), m.syms); + auto margs = msym.args; + auto msec = find(id_equals(msym.section), m.secs); + auto explicit_arg = kern._args.begin(); + + for (auto &marg : margs) { + switch (marg.semantic) { + case module::argument::general: + (*(explicit_arg++))->bind(*this, marg); + break; + + case module::argument::grid_dimension: { + const cl_uint dimension = grid_offset.size(); + auto arg = argument::create(marg); + + arg->set(sizeof(dimension), &dimension); + arg->bind(*this, marg); + break; + } + case module::argument::grid_offset: { + for (cl_uint x : pad_vector(*q, grid_offset, 0)) { + auto arg = argument::create(marg); + + arg->set(sizeof(x), &x); + arg->bind(*this, marg); + } + break; + } + case module::argument::image_size: { + auto img = dynamic_cast<image_argument &>(**(explicit_arg - 1)).get(); + std::vector<cl_uint> image_size{ + static_cast<cl_uint>(img->width()), + static_cast<cl_uint>(img->height()), + static_cast<cl_uint>(img->depth())}; + for (auto x : image_size) { + auto arg = argument::create(marg); + + arg->set(sizeof(x), &x); + arg->bind(*this, marg); + } + break; + } + case module::argument::image_format: { + auto img = dynamic_cast<image_argument &>(**(explicit_arg - 1)).get(); + cl_image_format fmt = img->format(); + std::vector<cl_uint> image_format{ + static_cast<cl_uint>(fmt.image_channel_data_type), + static_cast<cl_uint>(fmt.image_channel_order)}; + for (auto x : image_format) { + auto arg = argument::create(marg); + + arg->set(sizeof(x), &x); + arg->bind(*this, marg); + } + break; + } + } + } + + // Create a new compute state if anything changed. + if (!st || q != _q || + cs.req_local_mem != mem_local || + cs.req_input_mem != input.size()) { + if (st) + _q->pipe->delete_compute_state(_q->pipe, st); + + cs.ir_type = q->device().ir_format(); + cs.prog = &(msec.data[0]); + cs.req_local_mem = mem_local; + cs.req_input_mem = input.size(); + st = q->pipe->create_compute_state(q->pipe, &cs); + if (!st) { + unbind(); // Cleanup + throw error(CL_OUT_OF_RESOURCES); + } + } + + return st; +} + +void +kernel::exec_context::unbind() { + for (auto &arg : kern.args()) + arg.unbind(*this); + + input.clear(); + samplers.clear(); + sviews.clear(); + resources.clear(); + g_buffers.clear(); + g_handles.clear(); + mem_local = 0; +} + +namespace { + template<typename T> + std::vector<uint8_t> + bytes(const T& x) { + return { (uint8_t *)&x, (uint8_t *)&x + sizeof(x) }; + } + + /// + /// Transform buffer \a v from the native byte order into the byte + /// order specified by \a e. + /// + template<typename T> + void + byteswap(T &v, pipe_endian e) { + if (PIPE_ENDIAN_NATIVE != e) + std::reverse(v.begin(), v.end()); + } + + /// + /// Pad buffer \a v to the next multiple of \a n. + /// + template<typename T> + void + align(T &v, size_t n) { + v.resize(util_align_npot(v.size(), n)); + } + + bool + msb(const std::vector<uint8_t> &s) { + if (PIPE_ENDIAN_NATIVE == PIPE_ENDIAN_LITTLE) + return s.back() & 0x80; + else + return s.front() & 0x80; + } + + /// + /// Resize buffer \a v to size \a n using sign or zero extension + /// according to \a ext. + /// + template<typename T> + void + extend(T &v, enum module::argument::ext_type ext, size_t n) { + const size_t m = std::min(v.size(), n); + const bool sign_ext = (ext == module::argument::sign_ext); + const uint8_t fill = (sign_ext && msb(v) ? ~0 : 0); + T w(n, fill); + + if (PIPE_ENDIAN_NATIVE == PIPE_ENDIAN_LITTLE) + std::copy_n(v.begin(), m, w.begin()); + else + std::copy_n(v.end() - m, m, w.end() - m); + + std::swap(v, w); + } + + /// + /// Append buffer \a w to \a v. + /// + template<typename T> + void + insert(T &v, const T &w) { + v.insert(v.end(), w.begin(), w.end()); + } + + /// + /// Append \a n elements to the end of buffer \a v. + /// + template<typename T> + size_t + allocate(T &v, size_t n) { + size_t pos = v.size(); + v.resize(pos + n); + return pos; + } +} + +std::unique_ptr<kernel::argument> +kernel::argument::create(const module::argument &marg) { + switch (marg.type) { + case module::argument::scalar: + return std::unique_ptr<kernel::argument>(new scalar_argument(marg.size)); + + case module::argument::global: + return std::unique_ptr<kernel::argument>(new global_argument); + + case module::argument::local: + return std::unique_ptr<kernel::argument>(new local_argument); + + case module::argument::constant: + return std::unique_ptr<kernel::argument>(new constant_argument); + + case module::argument::image2d_rd: + case module::argument::image3d_rd: + return std::unique_ptr<kernel::argument>(new image_rd_argument); + + case module::argument::image2d_wr: + case module::argument::image3d_wr: + return std::unique_ptr<kernel::argument>(new image_wr_argument); + + case module::argument::sampler: + return std::unique_ptr<kernel::argument>(new sampler_argument); + + } + throw error(CL_INVALID_KERNEL_DEFINITION); +} + +kernel::argument::argument() : _set(false) { +} + +bool +kernel::argument::set() const { + return _set; +} + +size_t +kernel::argument::storage() const { + return 0; +} + +kernel::scalar_argument::scalar_argument(size_t size) : size(size) { +} + +void +kernel::scalar_argument::set(size_t size, const void *value) { + if (!value) + throw error(CL_INVALID_ARG_VALUE); + + if (size != this->size) + throw error(CL_INVALID_ARG_SIZE); + + v = { (uint8_t *)value, (uint8_t *)value + size }; + _set = true; +} + +void +kernel::scalar_argument::bind(exec_context &ctx, + const module::argument &marg) { + auto w = v; + + extend(w, marg.ext_type, marg.target_size); + byteswap(w, ctx.q->device().endianness()); + align(ctx.input, marg.target_align); + insert(ctx.input, w); +} + +void +kernel::scalar_argument::unbind(exec_context &ctx) { +} + +void +kernel::global_argument::set(size_t size, const void *value) { + if (size != sizeof(cl_mem)) + throw error(CL_INVALID_ARG_SIZE); + + buf = pobj<buffer>(value ? *(cl_mem *)value : NULL); + svm = nullptr; + _set = true; +} + +void +kernel::global_argument::set_svm(const void *value) { + svm = value; + buf = nullptr; + _set = true; +} + +void +kernel::global_argument::bind(exec_context &ctx, + const module::argument &marg) { + align(ctx.input, marg.target_align); + + if (buf) { + const resource &r = buf->resource(*ctx.q); + ctx.g_handles.push_back(ctx.input.size()); + ctx.g_buffers.push_back(r.pipe); + + // How to handle multi-demensional offsets? + // We don't need to. Buffer offsets are always + // one-dimensional. + auto v = bytes(r.offset[0]); + extend(v, marg.ext_type, marg.target_size); + byteswap(v, ctx.q->device().endianness()); + insert(ctx.input, v); + } else if (svm) { + auto v = bytes(svm); + extend(v, marg.ext_type, marg.target_size); + byteswap(v, ctx.q->device().endianness()); + insert(ctx.input, v); + } else { + // Null pointer. + allocate(ctx.input, marg.target_size); + } +} + +void +kernel::global_argument::unbind(exec_context &ctx) { +} + +size_t +kernel::local_argument::storage() const { + return _storage; +} + +void +kernel::local_argument::set(size_t size, const void *value) { + if (value) + throw error(CL_INVALID_ARG_VALUE); + + if (!size) + throw error(CL_INVALID_ARG_SIZE); + + _storage = size; + _set = true; +} + +void +kernel::local_argument::bind(exec_context &ctx, + const module::argument &marg) { + auto v = bytes(ctx.mem_local); + + extend(v, module::argument::zero_ext, marg.target_size); + byteswap(v, ctx.q->device().endianness()); + align(ctx.input, marg.target_align); + insert(ctx.input, v); + + ctx.mem_local += _storage; +} + +void +kernel::local_argument::unbind(exec_context &ctx) { +} + +void +kernel::constant_argument::set(size_t size, const void *value) { + if (size != sizeof(cl_mem)) + throw error(CL_INVALID_ARG_SIZE); + + buf = pobj<buffer>(value ? *(cl_mem *)value : NULL); + _set = true; +} + +void +kernel::constant_argument::bind(exec_context &ctx, + const module::argument &marg) { + align(ctx.input, marg.target_align); + + if (buf) { + resource &r = buf->resource(*ctx.q); + auto v = bytes(ctx.resources.size() << 24 | r.offset[0]); + + extend(v, module::argument::zero_ext, marg.target_size); + byteswap(v, ctx.q->device().endianness()); + insert(ctx.input, v); + + st = r.bind_surface(*ctx.q, false); + ctx.resources.push_back(st); + } else { + // Null pointer. + allocate(ctx.input, marg.target_size); + } +} + +void +kernel::constant_argument::unbind(exec_context &ctx) { + if (buf) + buf->resource(*ctx.q).unbind_surface(*ctx.q, st); +} + +void +kernel::image_rd_argument::set(size_t size, const void *value) { + if (!value) + throw error(CL_INVALID_ARG_VALUE); + + if (size != sizeof(cl_mem)) + throw error(CL_INVALID_ARG_SIZE); + + img = &obj<image>(*(cl_mem *)value); + _set = true; +} + +void +kernel::image_rd_argument::bind(exec_context &ctx, + const module::argument &marg) { + auto v = bytes(ctx.sviews.size()); + + extend(v, module::argument::zero_ext, marg.target_size); + byteswap(v, ctx.q->device().endianness()); + align(ctx.input, marg.target_align); + insert(ctx.input, v); + + st = img->resource(*ctx.q).bind_sampler_view(*ctx.q); + ctx.sviews.push_back(st); +} + +void +kernel::image_rd_argument::unbind(exec_context &ctx) { + img->resource(*ctx.q).unbind_sampler_view(*ctx.q, st); +} + +void +kernel::image_wr_argument::set(size_t size, const void *value) { + if (!value) + throw error(CL_INVALID_ARG_VALUE); + + if (size != sizeof(cl_mem)) + throw error(CL_INVALID_ARG_SIZE); + + img = &obj<image>(*(cl_mem *)value); + _set = true; +} + +void +kernel::image_wr_argument::bind(exec_context &ctx, + const module::argument &marg) { + auto v = bytes(ctx.resources.size()); + + extend(v, module::argument::zero_ext, marg.target_size); + byteswap(v, ctx.q->device().endianness()); + align(ctx.input, marg.target_align); + insert(ctx.input, v); + + st = img->resource(*ctx.q).bind_surface(*ctx.q, true); + ctx.resources.push_back(st); +} + +void +kernel::image_wr_argument::unbind(exec_context &ctx) { + img->resource(*ctx.q).unbind_surface(*ctx.q, st); +} + +void +kernel::sampler_argument::set(size_t size, const void *value) { + if (!value) + throw error(CL_INVALID_SAMPLER); + + if (size != sizeof(cl_sampler)) + throw error(CL_INVALID_ARG_SIZE); + + s = &obj(*(cl_sampler *)value); + _set = true; +} + +void +kernel::sampler_argument::bind(exec_context &ctx, + const module::argument &marg) { + st = s->bind(*ctx.q); + ctx.samplers.push_back(st); +} + +void +kernel::sampler_argument::unbind(exec_context &ctx) { + s->unbind(*ctx.q, st); +} diff --git a/src/gallium/frontends/clover/core/kernel.hpp b/src/gallium/frontends/clover/core/kernel.hpp new file mode 100644 index 00000000000..4441091f300 --- /dev/null +++ b/src/gallium/frontends/clover/core/kernel.hpp @@ -0,0 +1,251 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_KERNEL_HPP +#define CLOVER_CORE_KERNEL_HPP + +#include <memory> + +#include "core/object.hpp" +#include "core/program.hpp" +#include "core/memory.hpp" +#include "core/sampler.hpp" +#include "pipe/p_state.h" + +namespace clover { + class kernel : public ref_counter, public _cl_kernel { + private: + /// + /// Class containing all the state required to execute a compute + /// kernel. + /// + struct exec_context { + exec_context(kernel &kern); + ~exec_context(); + + exec_context(const exec_context &) = delete; + exec_context & + operator=(const exec_context &) = delete; + + void *bind(intrusive_ptr<command_queue> _q, + const std::vector<size_t> &grid_offset); + void unbind(); + + kernel &kern; + intrusive_ptr<command_queue> q; + + std::vector<uint8_t> input; + std::vector<void *> samplers; + std::vector<pipe_sampler_view *> sviews; + std::vector<pipe_surface *> resources; + std::vector<pipe_resource *> g_buffers; + std::vector<size_t> g_handles; + size_t mem_local; + + private: + void *st; + pipe_compute_state cs; + }; + + public: + class argument { + public: + static std::unique_ptr<argument> + create(const module::argument &marg); + + argument(const argument &arg) = delete; + argument & + operator=(const argument &arg) = delete; + + /// \a true if the argument has been set. + bool set() const; + + /// Storage space required for the referenced object. + virtual size_t storage() const; + + /// Set this argument to some object. + virtual void set(size_t size, const void *value) = 0; + + /// Set this argument to an SVM pointer. + virtual void set_svm(const void *value) { + throw error(CL_INVALID_ARG_INDEX); + }; + + /// Allocate the necessary resources to bind the specified + /// object to this argument, and update \a ctx accordingly. + virtual void bind(exec_context &ctx, + const module::argument &marg) = 0; + + /// Free any resources that were allocated in bind(). + virtual void unbind(exec_context &ctx) = 0; + + virtual ~argument() {}; + protected: + argument(); + + bool _set; + }; + + private: + typedef adaptor_range< + derefs, std::vector<std::unique_ptr<argument>> & + > argument_range; + + typedef adaptor_range< + derefs, const std::vector<std::unique_ptr<argument>> & + > const_argument_range; + + public: + kernel(clover::program &prog, const std::string &name, + const std::vector<clover::module::argument> &margs); + + kernel(const kernel &kern) = delete; + kernel & + operator=(const kernel &kern) = delete; + + void launch(command_queue &q, + const std::vector<size_t> &grid_offset, + const std::vector<size_t> &grid_size, + const std::vector<size_t> &block_size); + + size_t mem_local() const; + size_t mem_private() const; + + const std::string &name() const; + + std::vector<size_t> + optimal_block_size(const command_queue &q, + const std::vector<size_t> &grid_size) const; + std::vector<size_t> + required_block_size() const; + + argument_range args(); + const_argument_range args() const; + + const intrusive_ref<clover::program> program; + + private: + const clover::module &module(const command_queue &q) const; + + class scalar_argument : public argument { + public: + scalar_argument(size_t size); + + virtual void set(size_t size, const void *value); + virtual void bind(exec_context &ctx, + const module::argument &marg); + virtual void unbind(exec_context &ctx); + + private: + size_t size; + std::vector<uint8_t> v; + }; + + class global_argument : public argument { + public: + virtual void set(size_t size, const void *value); + virtual void set_svm(const void *value); + virtual void bind(exec_context &ctx, + const module::argument &marg); + virtual void unbind(exec_context &ctx); + + private: + buffer *buf; + const void *svm; + }; + + class local_argument : public argument { + public: + virtual size_t storage() const; + + virtual void set(size_t size, const void *value); + virtual void bind(exec_context &ctx, + const module::argument &marg); + virtual void unbind(exec_context &ctx); + + private: + size_t _storage = 0; + }; + + class constant_argument : public argument { + public: + virtual void set(size_t size, const void *value); + virtual void bind(exec_context &ctx, + const module::argument &marg); + virtual void unbind(exec_context &ctx); + + private: + buffer *buf; + pipe_surface *st; + }; + + class image_argument : public argument { + public: + const image *get() const { + return img; + } + protected: + image *img; + }; + + class image_rd_argument : public image_argument { + public: + virtual void set(size_t size, const void *value); + virtual void bind(exec_context &ctx, + const module::argument &marg); + virtual void unbind(exec_context &ctx); + + private: + pipe_sampler_view *st; + }; + + class image_wr_argument : public image_argument { + public: + virtual void set(size_t size, const void *value); + virtual void bind(exec_context &ctx, + const module::argument &marg); + virtual void unbind(exec_context &ctx); + + private: + pipe_surface *st; + }; + + class sampler_argument : public argument { + public: + virtual void set(size_t size, const void *value); + virtual void bind(exec_context &ctx, + const module::argument &marg); + virtual void unbind(exec_context &ctx); + + private: + sampler *s; + void *st; + }; + + std::vector<std::unique_ptr<argument>> _args; + std::string _name; + exec_context exec; + const ref_holder program_ref; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/memory.cpp b/src/gallium/frontends/clover/core/memory.cpp new file mode 100644 index 00000000000..ed13d92c281 --- /dev/null +++ b/src/gallium/frontends/clover/core/memory.cpp @@ -0,0 +1,214 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/memory.hpp" +#include "core/resource.hpp" +#include "util/format/u_format.h" + +using namespace clover; + +memory_obj::memory_obj(clover::context &ctx, cl_mem_flags flags, + size_t size, void *host_ptr) : + context(ctx), _flags(flags), + _size(size), _host_ptr(host_ptr) { + if (flags & CL_MEM_COPY_HOST_PTR) + data.append((char *)host_ptr, size); +} + +memory_obj::~memory_obj() { + while (_destroy_notify.size()) { + _destroy_notify.top()(); + _destroy_notify.pop(); + } +} + +bool +memory_obj::operator==(const memory_obj &obj) const { + return this == &obj; +} + +void +memory_obj::destroy_notify(std::function<void ()> f) { + _destroy_notify.push(f); +} + +cl_mem_flags +memory_obj::flags() const { + return _flags; +} + +size_t +memory_obj::size() const { + return _size; +} + +void * +memory_obj::host_ptr() const { + return _host_ptr; +} + +buffer::buffer(clover::context &ctx, cl_mem_flags flags, + size_t size, void *host_ptr) : + memory_obj(ctx, flags, size, host_ptr) { +} + +cl_mem_object_type +buffer::type() const { + return CL_MEM_OBJECT_BUFFER; +} + +root_buffer::root_buffer(clover::context &ctx, cl_mem_flags flags, + size_t size, void *host_ptr) : + buffer(ctx, flags, size, host_ptr) { +} + +resource & +root_buffer::resource(command_queue &q) { + // Create a new resource if there's none for this device yet. + if (!resources.count(&q.device())) { + auto r = (!resources.empty() ? + new root_resource(q.device(), *this, + *resources.begin()->second) : + new root_resource(q.device(), *this, q, data)); + + resources.insert(std::make_pair(&q.device(), + std::unique_ptr<root_resource>(r))); + data.clear(); + } + + return *resources.find(&q.device())->second; +} + +sub_buffer::sub_buffer(root_buffer &parent, cl_mem_flags flags, + size_t offset, size_t size) : + buffer(parent.context(), flags, size, + (char *)parent.host_ptr() + offset), + parent(parent), _offset(offset) { +} + +resource & +sub_buffer::resource(command_queue &q) { + // Create a new resource if there's none for this device yet. + if (!resources.count(&q.device())) { + auto r = new sub_resource(parent().resource(q), {{ offset() }}); + + resources.insert(std::make_pair(&q.device(), + std::unique_ptr<sub_resource>(r))); + } + + return *resources.find(&q.device())->second; +} + +size_t +sub_buffer::offset() const { + return _offset; +} + +image::image(clover::context &ctx, cl_mem_flags flags, + const cl_image_format *format, + size_t width, size_t height, size_t depth, + size_t row_pitch, size_t slice_pitch, size_t size, + void *host_ptr) : + memory_obj(ctx, flags, size, host_ptr), + _format(*format), _width(width), _height(height), _depth(depth), + _row_pitch(row_pitch), _slice_pitch(slice_pitch) { +} + +resource & +image::resource(command_queue &q) { + // Create a new resource if there's none for this device yet. + if (!resources.count(&q.device())) { + auto r = (!resources.empty() ? + new root_resource(q.device(), *this, + *resources.begin()->second) : + new root_resource(q.device(), *this, q, data)); + + resources.insert(std::make_pair(&q.device(), + std::unique_ptr<root_resource>(r))); + data.clear(); + } + + return *resources.find(&q.device())->second; +} + +cl_image_format +image::format() const { + return _format; +} + +size_t +image::width() const { + return _width; +} + +size_t +image::height() const { + return _height; +} + +size_t +image::depth() const { + return _depth; +} + +size_t +image::pixel_size() const { + return util_format_get_blocksize(translate_format(_format)); +} + +size_t +image::row_pitch() const { + return _row_pitch; +} + +size_t +image::slice_pitch() const { + return _slice_pitch; +} + +image2d::image2d(clover::context &ctx, cl_mem_flags flags, + const cl_image_format *format, size_t width, + size_t height, size_t row_pitch, + void *host_ptr) : + image(ctx, flags, format, width, height, 1, + row_pitch, 0, height * row_pitch, host_ptr) { +} + +cl_mem_object_type +image2d::type() const { + return CL_MEM_OBJECT_IMAGE2D; +} + +image3d::image3d(clover::context &ctx, cl_mem_flags flags, + const cl_image_format *format, + size_t width, size_t height, size_t depth, + size_t row_pitch, size_t slice_pitch, + void *host_ptr) : + image(ctx, flags, format, width, height, depth, + row_pitch, slice_pitch, depth * slice_pitch, + host_ptr) { +} + +cl_mem_object_type +image3d::type() const { + return CL_MEM_OBJECT_IMAGE3D; +} diff --git a/src/gallium/frontends/clover/core/memory.hpp b/src/gallium/frontends/clover/core/memory.hpp new file mode 100644 index 00000000000..bd6da6be4d1 --- /dev/null +++ b/src/gallium/frontends/clover/core/memory.hpp @@ -0,0 +1,159 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_MEMORY_HPP +#define CLOVER_CORE_MEMORY_HPP + +#include <functional> +#include <map> +#include <memory> +#include <stack> + +#include "core/object.hpp" +#include "core/queue.hpp" +#include "core/resource.hpp" + +namespace clover { + class memory_obj : public ref_counter, public _cl_mem { + protected: + memory_obj(clover::context &ctx, cl_mem_flags flags, + size_t size, void *host_ptr); + + memory_obj(const memory_obj &obj) = delete; + memory_obj & + operator=(const memory_obj &obj) = delete; + + public: + virtual ~memory_obj(); + + bool + operator==(const memory_obj &obj) const; + + virtual cl_mem_object_type type() const = 0; + virtual clover::resource &resource(command_queue &q) = 0; + + void destroy_notify(std::function<void ()> f); + cl_mem_flags flags() const; + size_t size() const; + void *host_ptr() const; + + const intrusive_ref<clover::context> context; + + private: + cl_mem_flags _flags; + size_t _size; + void *_host_ptr; + std::stack<std::function<void ()>> _destroy_notify; + + protected: + std::string data; + }; + + class buffer : public memory_obj { + protected: + buffer(clover::context &ctx, cl_mem_flags flags, + size_t size, void *host_ptr); + + public: + virtual cl_mem_object_type type() const; + }; + + class root_buffer : public buffer { + public: + root_buffer(clover::context &ctx, cl_mem_flags flags, + size_t size, void *host_ptr); + + virtual clover::resource &resource(command_queue &q); + + private: + std::map<device *, + std::unique_ptr<root_resource>> resources; + }; + + class sub_buffer : public buffer { + public: + sub_buffer(root_buffer &parent, cl_mem_flags flags, + size_t offset, size_t size); + + virtual clover::resource &resource(command_queue &q); + size_t offset() const; + + const intrusive_ref<root_buffer> parent; + + private: + size_t _offset; + std::map<device *, + std::unique_ptr<sub_resource>> resources; + }; + + class image : public memory_obj { + protected: + image(clover::context &ctx, cl_mem_flags flags, + const cl_image_format *format, + size_t width, size_t height, size_t depth, + size_t row_pitch, size_t slice_pitch, size_t size, + void *host_ptr); + + public: + virtual clover::resource &resource(command_queue &q); + cl_image_format format() const; + size_t width() const; + size_t height() const; + size_t depth() const; + size_t pixel_size() const; + size_t row_pitch() const; + size_t slice_pitch() const; + + private: + cl_image_format _format; + size_t _width; + size_t _height; + size_t _depth; + size_t _row_pitch; + size_t _slice_pitch; + std::map<device *, + std::unique_ptr<root_resource>> resources; + }; + + class image2d : public image { + public: + image2d(clover::context &ctx, cl_mem_flags flags, + const cl_image_format *format, size_t width, + size_t height, size_t row_pitch, + void *host_ptr); + + virtual cl_mem_object_type type() const; + }; + + class image3d : public image { + public: + image3d(clover::context &ctx, cl_mem_flags flags, + const cl_image_format *format, + size_t width, size_t height, size_t depth, + size_t row_pitch, size_t slice_pitch, + void *host_ptr); + + virtual cl_mem_object_type type() const; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/module.cpp b/src/gallium/frontends/clover/core/module.cpp new file mode 100644 index 00000000000..a6c5b98d8e0 --- /dev/null +++ b/src/gallium/frontends/clover/core/module.cpp @@ -0,0 +1,228 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include <type_traits> +#include <iostream> + +#include "core/module.hpp" + +using namespace clover; + +namespace { + template<typename T, typename = void> + struct _serializer; + + /// Serialize the specified object. + template<typename T> + void + _proc(std::ostream &os, const T &x) { + _serializer<T>::proc(os, x); + } + + /// Deserialize the specified object. + template<typename T> + void + _proc(std::istream &is, T &x) { + _serializer<T>::proc(is, x); + } + + template<typename T> + T + _proc(std::istream &is) { + T x; + _serializer<T>::proc(is, x); + return x; + } + + /// Calculate the size of the specified object. + template<typename T> + void + _proc(module::size_t &sz, const T &x) { + _serializer<T>::proc(sz, x); + } + + /// (De)serialize a scalar value. + template<typename T> + struct _serializer<T, typename std::enable_if< + std::is_scalar<T>::value>::type> { + static void + proc(std::ostream &os, const T &x) { + os.write(reinterpret_cast<const char *>(&x), sizeof(x)); + } + + static void + proc(std::istream &is, T &x) { + is.read(reinterpret_cast<char *>(&x), sizeof(x)); + } + + static void + proc(module::size_t &sz, const T &x) { + sz += sizeof(x); + } + }; + + /// (De)serialize a vector. + template<typename T> + struct _serializer<std::vector<T>, + typename std::enable_if< + !std::is_scalar<T>::value>::type> { + static void + proc(std::ostream &os, const std::vector<T> &v) { + _proc<uint32_t>(os, v.size()); + + for (size_t i = 0; i < v.size(); i++) + _proc<T>(os, v[i]); + } + + static void + proc(std::istream &is, std::vector<T> &v) { + v.resize(_proc<uint32_t>(is)); + + for (size_t i = 0; i < v.size(); i++) + new(&v[i]) T(_proc<T>(is)); + } + + static void + proc(module::size_t &sz, const std::vector<T> &v) { + sz += sizeof(uint32_t); + + for (size_t i = 0; i < v.size(); i++) + _proc<T>(sz, v[i]); + } + }; + + template<typename T> + struct _serializer<std::vector<T>, + typename std::enable_if< + std::is_scalar<T>::value>::type> { + static void + proc(std::ostream &os, const std::vector<T> &v) { + _proc<uint32_t>(os, v.size()); + os.write(reinterpret_cast<const char *>(&v[0]), + v.size() * sizeof(T)); + } + + static void + proc(std::istream &is, std::vector<T> &v) { + v.resize(_proc<uint32_t>(is)); + is.read(reinterpret_cast<char *>(&v[0]), + v.size() * sizeof(T)); + } + + static void + proc(module::size_t &sz, const std::vector<T> &v) { + sz += sizeof(uint32_t) + sizeof(T) * v.size(); + } + }; + + /// (De)serialize a string. + template<> + struct _serializer<std::string> { + static void + proc(std::ostream &os, const std::string &s) { + _proc<uint32_t>(os, s.size()); + os.write(&s[0], s.size() * sizeof(std::string::value_type)); + } + + static void + proc(std::istream &is, std::string &s) { + s.resize(_proc<uint32_t>(is)); + is.read(&s[0], s.size() * sizeof(std::string::value_type)); + } + + static void + proc(module::size_t &sz, const std::string &s) { + sz += sizeof(uint32_t) + sizeof(std::string::value_type) * s.size(); + } + }; + + /// (De)serialize a module::section. + template<> + struct _serializer<module::section> { + template<typename S, typename QT> + static void + proc(S &s, QT &x) { + _proc(s, x.id); + _proc(s, x.type); + _proc(s, x.size); + _proc(s, x.data); + } + }; + + /// (De)serialize a module::argument. + template<> + struct _serializer<module::argument> { + template<typename S, typename QT> + static void + proc(S &s, QT &x) { + _proc(s, x.type); + _proc(s, x.size); + _proc(s, x.target_size); + _proc(s, x.target_align); + _proc(s, x.ext_type); + _proc(s, x.semantic); + } + }; + + /// (De)serialize a module::symbol. + template<> + struct _serializer<module::symbol> { + template<typename S, typename QT> + static void + proc(S &s, QT &x) { + _proc(s, x.name); + _proc(s, x.section); + _proc(s, x.offset); + _proc(s, x.args); + } + }; + + /// (De)serialize a module. + template<> + struct _serializer<module> { + template<typename S, typename QT> + static void + proc(S &s, QT &x) { + _proc(s, x.syms); + _proc(s, x.secs); + } + }; +}; + +namespace clover { + void + module::serialize(std::ostream &os) const { + _proc(os, *this); + } + + module + module::deserialize(std::istream &is) { + return _proc<module>(is); + } + + module::size_t + module::size() const { + size_t sz = 0; + _proc(sz, *this); + return sz; + } +} diff --git a/src/gallium/frontends/clover/core/module.hpp b/src/gallium/frontends/clover/core/module.hpp new file mode 100644 index 00000000000..2ddd26426fb --- /dev/null +++ b/src/gallium/frontends/clover/core/module.hpp @@ -0,0 +1,128 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_MODULE_HPP +#define CLOVER_CORE_MODULE_HPP + +#include <vector> +#include <string> + +namespace clover { + struct module { + typedef uint32_t resource_id; + typedef uint32_t size_t; + + struct section { + enum type { + text_intermediate, + text_library, + text_executable, + data_constant, + data_global, + data_local, + data_private + }; + + section(resource_id id, enum type type, size_t size, + const std::vector<char> &data) : + id(id), type(type), size(size), data(data) { } + section() : id(0), type(text_intermediate), size(0), data() { } + + resource_id id; + type type; + size_t size; + std::vector<char> data; + }; + + struct argument { + enum type { + scalar, + constant, + global, + local, + image2d_rd, + image2d_wr, + image3d_rd, + image3d_wr, + sampler + }; + + enum ext_type { + zero_ext, + sign_ext + }; + + enum semantic { + general, + grid_dimension, + grid_offset, + image_size, + image_format + }; + + argument(enum type type, size_t size, + size_t target_size, size_t target_align, + enum ext_type ext_type, + enum semantic semantic = general) : + type(type), size(size), + target_size(target_size), target_align(target_align), + ext_type(ext_type), semantic(semantic) { } + + argument(enum type type, size_t size) : + type(type), size(size), + target_size(size), target_align(1), + ext_type(zero_ext), semantic(general) { } + + argument() : type(scalar), size(0), + target_size(0), target_align(1), + ext_type(zero_ext), semantic(general) { } + + type type; + size_t size; + size_t target_size; + size_t target_align; + ext_type ext_type; + semantic semantic; + }; + + struct symbol { + symbol(const std::string &name, resource_id section, + size_t offset, const std::vector<argument> &args) : + name(name), section(section), offset(offset), args(args) { } + symbol() : name(), section(0), offset(0), args() { } + + std::string name; + resource_id section; + size_t offset; + std::vector<argument> args; + }; + + void serialize(std::ostream &os) const; + static module deserialize(std::istream &is); + size_t size() const; + + std::vector<symbol> syms; + std::vector<section> secs; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/object.hpp b/src/gallium/frontends/clover/core/object.hpp new file mode 100644 index 00000000000..8fc2175d236 --- /dev/null +++ b/src/gallium/frontends/clover/core/object.hpp @@ -0,0 +1,239 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_OBJECT_HPP +#define CLOVER_CORE_OBJECT_HPP + +#include <cassert> +#include <functional> +#include <vector> + +#include "CL/cl.h" + +#include "core/error.hpp" +#include "core/property.hpp" +#include "api/dispatch.hpp" +#include "util/macros.h" + +/// +/// Main namespace of the CL state tracker. +/// +namespace clover { + /// + /// Class that represents a CL API object. + /// + template<typename T, typename S> + struct descriptor { + typedef T object_type; + typedef S descriptor_type; + + descriptor() : dispatch(&_dispatch) { + static_assert(std::is_standard_layout<descriptor_type>::value, + "ICD requires CL API objects to be standard layout."); + } + + const cl_icd_dispatch *dispatch; + }; + + struct default_tag; + struct allow_empty_tag; + struct wait_list_tag; + struct property_list_tag; + + namespace detail { + template<typename T, typename D> + struct descriptor_traits { + typedef T object_type; + + static void + validate(D *d) { + auto o = static_cast<typename D::object_type *>(d); + if (!o || o->dispatch != &_dispatch || + !dynamic_cast<object_type *>(o)) + throw invalid_object_error<T>(); + } + + static void + validate_list(D * const *ds, size_t n) { + if (!ds || !n) + throw error(CL_INVALID_VALUE); + } + }; + + template<typename D> + struct descriptor_traits<default_tag, D> { + typedef typename D::object_type object_type; + + static void + validate(D *d) { + if (!d || d->dispatch != &_dispatch) + throw invalid_object_error<object_type>(); + } + + static void + validate_list(D *const *ds, size_t n) { + if (!ds || !n) + throw error(CL_INVALID_VALUE); + } + }; + + template<typename D> + struct descriptor_traits<allow_empty_tag, D> { + typedef typename D::object_type object_type; + + static void + validate(D *d) { + if (!d || d->dispatch != &_dispatch) + throw invalid_object_error<object_type>(); + } + + static void + validate_list(D *const *ds, size_t n) { + if (bool(ds) != bool(n)) + throw error(CL_INVALID_VALUE); + } + }; + + template<typename D> + struct descriptor_traits<wait_list_tag, D> { + typedef typename D::object_type object_type; + + static void + validate(D *d) { + if (!d || d->dispatch != &_dispatch) + throw invalid_wait_list_error(); + } + + static void + validate_list(D *const *ds, size_t n) { + if (bool(ds) != bool(n)) + throw invalid_wait_list_error(); + } + }; + } + + /// + /// Get a Clover object from an API object performing object + /// validation. + /// + /// \a T can either be the Clover object type to return or a \a tag + /// object to select some special validation behavior by means of a + /// specialization of the detail::descriptor_traits template. The + /// default behavior is to infer the most general Clover object + /// type for the given API object. + /// + template<typename T = default_tag, typename D> + typename detail::descriptor_traits<T, D>::object_type & + obj(D *d) { + detail::descriptor_traits<T, D>::validate(d); + + return static_cast< + typename detail::descriptor_traits<T, D>::object_type &>(*d); + } + + /// + /// Get a pointer to a Clover object from an API object performing + /// object validation. Returns \c NULL if its argument is \c NULL. + /// + /// \sa obj + /// + template<typename T = default_tag, typename D> + typename detail::descriptor_traits<T, D>::object_type * + pobj(D *d) { + if (d) + detail::descriptor_traits<T, D>::validate(d); + + return static_cast< + typename detail::descriptor_traits<T, D>::object_type *>(d); + } + + /// + /// Get an API object from a Clover object. + /// + template<typename O> + typename O::descriptor_type * + desc(O &o) { + return static_cast<typename O::descriptor_type *>(&o); + } + + /// + /// Get an API object from a pointer to a Clover object. + /// + template<typename O> + typename O::descriptor_type * + desc(O *o) { + return static_cast<typename O::descriptor_type *>(o); + } + + /// + /// Get a range of Clover objects from a range of API objects + /// performing object validation. + /// + /// \sa obj + /// + template<typename T = default_tag, typename D> + ref_vector<typename detail::descriptor_traits<T, D>::object_type> + objs(D *const *ds, size_t n) { + detail::descriptor_traits<T, D>::validate_list(ds, n); + return map(obj<T, D>, range(ds, n)); + } + + /// + /// Get a range of API objects from a range of Clover objects. + /// + template<typename Os> + std::vector<typename Os::value_type::descriptor_type *> + descs(const Os &os) { + return map([](typename Os::value_type &o) { + return desc(o); + }, os); + } +} + +struct _cl_context : + public clover::descriptor<clover::context, _cl_context> {}; + +struct _cl_device_id : + public clover::descriptor<clover::device, _cl_device_id> {}; + +struct _cl_event : + public clover::descriptor<clover::event, _cl_event> {}; + +struct _cl_kernel : + public clover::descriptor<clover::kernel, _cl_kernel> {}; + +struct _cl_mem : + public clover::descriptor<clover::memory_obj, _cl_mem> {}; + +struct _cl_platform_id : + public clover::descriptor<clover::platform, _cl_platform_id> {}; + +struct _cl_program : + public clover::descriptor<clover::program, _cl_program> {}; + +struct _cl_command_queue : + public clover::descriptor<clover::command_queue, _cl_command_queue> {}; + +struct _cl_sampler : + public clover::descriptor<clover::sampler, _cl_sampler> {}; + +#endif diff --git a/src/gallium/frontends/clover/core/platform.cpp b/src/gallium/frontends/clover/core/platform.cpp new file mode 100644 index 00000000000..ddd63fc5a0d --- /dev/null +++ b/src/gallium/frontends/clover/core/platform.cpp @@ -0,0 +1,46 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/platform.hpp" + +using namespace clover; + +platform::platform() : adaptor_range(evals(), devs) { + int n = pipe_loader_probe(NULL, 0); + std::vector<pipe_loader_device *> ldevs(n); + + pipe_loader_probe(&ldevs.front(), n); + + for (pipe_loader_device *ldev : ldevs) { + try { + if (ldev) + devs.push_back(create<device>(*this, ldev)); + } catch (error &) { + pipe_loader_release(&ldev, 1); + } + } +} + +std::string +platform::supported_extensions() const { + return "cl_khr_icd"; +} diff --git a/src/gallium/frontends/clover/core/platform.hpp b/src/gallium/frontends/clover/core/platform.hpp new file mode 100644 index 00000000000..b94434c983c --- /dev/null +++ b/src/gallium/frontends/clover/core/platform.hpp @@ -0,0 +1,50 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_PLATFORM_HPP +#define CLOVER_CORE_PLATFORM_HPP + +#include <vector> + +#include "core/object.hpp" +#include "core/device.hpp" +#include "util/range.hpp" + +namespace clover { + class platform : public _cl_platform_id, + public adaptor_range< + evals, std::vector<intrusive_ref<device>> &> { + public: + platform(); + + platform(const platform &platform) = delete; + platform & + operator=(const platform &platform) = delete; + + std::string supported_extensions() const; + + protected: + std::vector<intrusive_ref<device>> devs; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/program.cpp b/src/gallium/frontends/clover/core/program.cpp new file mode 100644 index 00000000000..526e06a26c3 --- /dev/null +++ b/src/gallium/frontends/clover/core/program.cpp @@ -0,0 +1,135 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/compiler.hpp" +#include "core/program.hpp" + +using namespace clover; + +program::program(clover::context &ctx, const std::string &source) : + has_source(true), context(ctx), _devices(ctx.devices()), _source(source), + _kernel_ref_counter(0) { +} + +program::program(clover::context &ctx, + const ref_vector<device> &devs, + const std::vector<module> &binaries) : + has_source(false), context(ctx), + _devices(devs), _kernel_ref_counter(0) { + for_each([&](device &dev, const module &bin) { + _builds[&dev] = { bin }; + }, + devs, binaries); +} + +void +program::compile(const ref_vector<device> &devs, const std::string &opts, + const header_map &headers) { + if (has_source) { + _devices = devs; + + for (auto &dev : devs) { + std::string log; + + try { + const module m = + compiler::compile_program(_source, headers, dev, opts, log); + _builds[&dev] = { m, opts, log }; + } catch (...) { + _builds[&dev] = { module(), opts, log }; + throw; + } + } + } +} + +void +program::link(const ref_vector<device> &devs, const std::string &opts, + const ref_vector<program> &progs) { + _devices = devs; + + for (auto &dev : devs) { + const std::vector<module> ms = map([&](const program &prog) { + return prog.build(dev).binary; + }, progs); + std::string log = _builds[&dev].log; + + try { + const module m = compiler::link_program(ms, dev, opts, log); + _builds[&dev] = { m, opts, log }; + } catch (...) { + _builds[&dev] = { module(), opts, log }; + throw; + } + } +} + +const std::string & +program::source() const { + return _source; +} + +program::device_range +program::devices() const { + return map(evals(), _devices); +} + +cl_build_status +program::build::status() const { + if (!binary.secs.empty()) + return CL_BUILD_SUCCESS; + else if (log.size()) + return CL_BUILD_ERROR; + else + return CL_BUILD_NONE; +} + +cl_program_binary_type +program::build::binary_type() const { + if (any_of(type_equals(module::section::text_intermediate), binary.secs)) + return CL_PROGRAM_BINARY_TYPE_COMPILED_OBJECT; + else if (any_of(type_equals(module::section::text_library), binary.secs)) + return CL_PROGRAM_BINARY_TYPE_LIBRARY; + else if (any_of(type_equals(module::section::text_executable), binary.secs)) + return CL_PROGRAM_BINARY_TYPE_EXECUTABLE; + else + return CL_PROGRAM_BINARY_TYPE_NONE; +} + +const struct program::build & +program::build(const device &dev) const { + static const struct build null; + return _builds.count(&dev) ? _builds.find(&dev)->second : null; +} + +const std::vector<module::symbol> & +program::symbols() const { + if (_builds.empty()) + throw error(CL_INVALID_PROGRAM_EXECUTABLE); + + return _builds.begin()->second.binary.syms; +} + +unsigned +program::kernel_ref_count() const { + return _kernel_ref_counter.ref_count(); +} diff --git a/src/gallium/frontends/clover/core/program.hpp b/src/gallium/frontends/clover/core/program.hpp new file mode 100644 index 00000000000..05964e78a79 --- /dev/null +++ b/src/gallium/frontends/clover/core/program.hpp @@ -0,0 +1,91 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_PROGRAM_HPP +#define CLOVER_CORE_PROGRAM_HPP + +#include <map> + +#include "core/object.hpp" +#include "core/context.hpp" +#include "core/module.hpp" + +namespace clover { + typedef std::vector<std::pair<std::string, std::string>> header_map; + + class program : public ref_counter, public _cl_program { + private: + typedef adaptor_range< + evals, const std::vector<intrusive_ref<device>> &> device_range; + + public: + program(clover::context &ctx, + const std::string &source); + program(clover::context &ctx, + const ref_vector<device> &devs = {}, + const std::vector<module> &binaries = {}); + + program(const program &prog) = delete; + program & + operator=(const program &prog) = delete; + + void compile(const ref_vector<device> &devs, const std::string &opts, + const header_map &headers = {}); + void link(const ref_vector<device> &devs, const std::string &opts, + const ref_vector<program> &progs); + + const bool has_source; + const std::string &source() const; + + device_range devices() const; + + struct build { + build(const module &m = {}, const std::string &opts = {}, + const std::string &log = {}) : binary(m), opts(opts), log(log) {} + + cl_build_status status() const; + cl_program_binary_type binary_type() const; + + module binary; + std::string opts; + std::string log; + }; + + const build &build(const device &dev) const; + + const std::vector<module::symbol> &symbols() const; + + unsigned kernel_ref_count() const; + + const intrusive_ref<clover::context> context; + + friend class kernel; + + private: + std::vector<intrusive_ref<device>> _devices; + std::map<const device *, struct build> _builds; + std::string _source; + ref_counter _kernel_ref_counter; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/property.hpp b/src/gallium/frontends/clover/core/property.hpp new file mode 100644 index 00000000000..7f8e17684d9 --- /dev/null +++ b/src/gallium/frontends/clover/core/property.hpp @@ -0,0 +1,261 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_PROPERTY_HPP +#define CLOVER_CORE_PROPERTY_HPP + +#include <map> + +#include "util/range.hpp" +#include "util/algorithm.hpp" + +namespace clover { + class property_buffer; + + namespace detail { + template<typename T> + class property_scalar { + public: + property_scalar(property_buffer &buf) : buf(buf) { + } + + inline property_scalar & + operator=(const T &x); + + private: + property_buffer &buf; + }; + + template<typename T> + class property_vector { + public: + property_vector(property_buffer &buf) : buf(buf) { + } + + template<typename S> + inline property_vector & + operator=(const S &v); + + private: + property_buffer &buf; + }; + + template<typename T> + class property_matrix { + public: + property_matrix(property_buffer &buf) : buf(buf) { + } + + template<typename S> + inline property_matrix & + operator=(const S &v); + + private: + property_buffer &buf; + }; + + class property_string { + public: + property_string(property_buffer &buf) : buf(buf) { + } + + inline property_string & + operator=(const std::string &v); + + private: + property_buffer &buf; + }; + }; + + /// + /// Return value buffer used by the CL property query functions. + /// + class property_buffer { + public: + property_buffer(void *r_buf, size_t size, size_t *r_size) : + r_buf(r_buf), size(size), r_size(r_size) { + } + + template<typename T> + detail::property_scalar<T> + as_scalar() { + return { *this }; + } + + template<typename T> + detail::property_vector<T> + as_vector() { + return { *this }; + } + + template<typename T> + detail::property_matrix<T> + as_matrix() { + return { *this }; + } + + detail::property_string + as_string() { + return { *this }; + } + + template<typename T> + iterator_range<T *> + allocate(size_t n) { + if (r_buf && size < n * sizeof(T)) + throw error(CL_INVALID_VALUE); + + if (r_size) + *r_size = n * sizeof(T); + + if (r_buf) + return range((T *)r_buf, n); + else + return { }; + } + + private: + void *const r_buf; + const size_t size; + size_t *const r_size; + }; + + namespace detail { + template<typename T> + inline property_scalar<T> & + property_scalar<T>::operator=(const T &x) { + auto r = buf.allocate<T>(1); + + if (!r.empty()) + r.front() = x; + + return *this; + } + + template<typename T> + template<typename S> + inline property_vector<T> & + property_vector<T>::operator=(const S &v) { + auto r = buf.allocate<T>(v.size()); + + if (!r.empty()) + copy(v, r.begin()); + + return *this; + } + + template<typename T> + template<typename S> + inline property_matrix<T> & + property_matrix<T>::operator=(const S &v) { + auto r = buf.allocate<T *>(v.size()); + + if (!r.empty()) + for_each([](typename S::value_type src, T *dst) { + if (dst) + copy(src, dst); + }, v, r); + + return *this; + } + + inline property_string & + property_string::operator=(const std::string &v) { + auto r = buf.allocate<char>(v.size() + 1); + + if (!r.empty()) + copy(range(v.begin(), r.size()), r.begin()); + + return *this; + } + }; + + template<typename T> + class property_element { + public: + property_element() : x() { + } + + property_element(T x) : x(x) { + } + + template<typename S> + S + as() const { + assert(sizeof(S) <= sizeof(T)); + return reinterpret_cast<S>(x); + } + + private: + T x; + }; + + template<typename D> + using property_list = std::map<D, property_element<D>>; + + struct property_list_tag; + + /// + /// Create a clover::property_list object from a zero-terminated + /// CL property list. + /// + template<typename T, typename D, + typename = typename std::enable_if< + std::is_same<T, property_list_tag>::value>::type> + property_list<D> + obj(const D *d_props) { + property_list<D> props; + + while (d_props && *d_props) { + auto key = *d_props++; + auto value = *d_props++; + + if (props.count(key)) + throw error(CL_INVALID_PROPERTY); + + props.insert({ key, value }); + } + + return props; + } + + /// + /// Create a zero-terminated CL property list from a + /// clover::property_list object. + /// + template<typename D> + std::vector<D> + desc(const property_list<D> &props) { + std::vector<D> d_props; + + for (auto &prop : props) { + d_props.push_back(prop.first); + d_props.push_back(prop.second.template as<D>()); + } + + d_props.push_back(0); + + return d_props; + } +} + +#endif diff --git a/src/gallium/frontends/clover/core/queue.cpp b/src/gallium/frontends/clover/core/queue.cpp new file mode 100644 index 00000000000..c91b97ad15e --- /dev/null +++ b/src/gallium/frontends/clover/core/queue.cpp @@ -0,0 +1,102 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/queue.hpp" +#include "core/event.hpp" +#include "pipe/p_screen.h" +#include "pipe/p_context.h" +#include "pipe/p_state.h" + +using namespace clover; + +namespace { + void + debug_notify_callback(void *data, + unsigned *id, + enum pipe_debug_type type, + const char *fmt, + va_list args) { + const command_queue *queue = (const command_queue *)data; + char buffer[1024]; + vsnprintf(buffer, sizeof(buffer), fmt, args); + queue->context().notify(buffer); + } +} + +command_queue::command_queue(clover::context &ctx, clover::device &dev, + cl_command_queue_properties props) : + context(ctx), device(dev), props(props) { + pipe = dev.pipe->context_create(dev.pipe, NULL, PIPE_CONTEXT_COMPUTE_ONLY); + if (!pipe) + throw error(CL_INVALID_DEVICE); + + if (ctx.notify) { + struct pipe_debug_callback cb; + memset(&cb, 0, sizeof(cb)); + cb.debug_message = &debug_notify_callback; + cb.data = this; + if (pipe->set_debug_callback) + pipe->set_debug_callback(pipe, &cb); + } +} + +command_queue::~command_queue() { + pipe->destroy(pipe); +} + +void +command_queue::flush() { + pipe_screen *screen = device().pipe; + pipe_fence_handle *fence = NULL; + + std::lock_guard<std::mutex> lock(queued_events_mutex); + if (!queued_events.empty()) { + pipe->flush(pipe, &fence, 0); + + while (!queued_events.empty() && + queued_events.front()().signalled()) { + queued_events.front()().fence(fence); + queued_events.pop_front(); + } + + screen->fence_reference(screen, &fence, NULL); + } +} + +cl_command_queue_properties +command_queue::properties() const { + return props; +} + +bool +command_queue::profiling_enabled() const { + return props & CL_QUEUE_PROFILING_ENABLE; +} + +void +command_queue::sequence(hard_event &ev) { + std::lock_guard<std::mutex> lock(queued_events_mutex); + if (!queued_events.empty()) + queued_events.back()().chain(ev); + + queued_events.push_back(ev); +} diff --git a/src/gallium/frontends/clover/core/queue.hpp b/src/gallium/frontends/clover/core/queue.hpp new file mode 100644 index 00000000000..bddb86c0e4c --- /dev/null +++ b/src/gallium/frontends/clover/core/queue.hpp @@ -0,0 +1,78 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_QUEUE_HPP +#define CLOVER_CORE_QUEUE_HPP + +#include <deque> +#include <mutex> + +#include "core/object.hpp" +#include "core/context.hpp" +#include "core/timestamp.hpp" +#include "pipe/p_context.h" + +namespace clover { + class resource; + class mapping; + class hard_event; + + class command_queue : public ref_counter, public _cl_command_queue { + public: + command_queue(clover::context &ctx, clover::device &dev, + cl_command_queue_properties props); + ~command_queue(); + + command_queue(const command_queue &q) = delete; + command_queue & + operator=(const command_queue &q) = delete; + + void flush(); + + cl_command_queue_properties properties() const; + bool profiling_enabled() const; + + const intrusive_ref<clover::context> context; + const intrusive_ref<clover::device> device; + + friend class resource; + friend class root_resource; + friend class mapping; + friend class hard_event; + friend class sampler; + friend class kernel; + friend class clover::timestamp::query; + friend class clover::timestamp::current; + + private: + /// Serialize a hardware event with respect to the previous ones, + /// and push it to the pending list. + void sequence(hard_event &ev); + + cl_command_queue_properties props; + pipe_context *pipe; + std::mutex queued_events_mutex; + std::deque<intrusive_ref<hard_event>> queued_events; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/resource.cpp b/src/gallium/frontends/clover/core/resource.cpp new file mode 100644 index 00000000000..dd207982588 --- /dev/null +++ b/src/gallium/frontends/clover/core/resource.cpp @@ -0,0 +1,231 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/resource.hpp" +#include "core/memory.hpp" +#include "pipe/p_screen.h" +#include "util/u_sampler.h" +#include "util/format/u_format.h" +#include "util/u_inlines.h" + +using namespace clover; + +namespace { + class box { + public: + box(const resource::vector &origin, const resource::vector &size) : + pipe({ (int)origin[0], (int16_t)origin[1], + (int16_t)origin[2], (int)size[0], + (int16_t)size[1], (int16_t)size[2] }) { + } + + operator const pipe_box *() { + return &pipe; + } + + protected: + pipe_box pipe; + }; +} + +resource::resource(clover::device &dev, memory_obj &obj) : + device(dev), obj(obj), pipe(NULL), offset() { +} + +resource::~resource() { +} + +void +resource::copy(command_queue &q, const vector &origin, const vector ®ion, + resource &src_res, const vector &src_origin) { + auto p = offset + origin; + + q.pipe->resource_copy_region(q.pipe, pipe, 0, p[0], p[1], p[2], + src_res.pipe, 0, + box(src_res.offset + src_origin, region)); +} + +void * +resource::add_map(command_queue &q, cl_map_flags flags, bool blocking, + const vector &origin, const vector ®ion) { + maps.emplace_back(q, *this, flags, blocking, origin, region); + return maps.back(); +} + +void +resource::del_map(void *p) { + erase_if([&](const mapping &m) { + return static_cast<void *>(m) == p; + }, maps); +} + +unsigned +resource::map_count() const { + return maps.size(); +} + +pipe_sampler_view * +resource::bind_sampler_view(command_queue &q) { + pipe_sampler_view info; + + u_sampler_view_default_template(&info, pipe, pipe->format); + return q.pipe->create_sampler_view(q.pipe, pipe, &info); +} + +void +resource::unbind_sampler_view(command_queue &q, + pipe_sampler_view *st) { + q.pipe->sampler_view_destroy(q.pipe, st); +} + +pipe_surface * +resource::bind_surface(command_queue &q, bool rw) { + pipe_surface info {}; + + info.format = pipe->format; + info.writable = rw; + + if (pipe->target == PIPE_BUFFER) + info.u.buf.last_element = pipe->width0 - 1; + + return q.pipe->create_surface(q.pipe, pipe, &info); +} + +void +resource::unbind_surface(command_queue &q, pipe_surface *st) { + q.pipe->surface_destroy(q.pipe, st); +} + +root_resource::root_resource(clover::device &dev, memory_obj &obj, + command_queue &q, const std::string &data) : + resource(dev, obj) { + pipe_resource info {}; + const bool user_ptr_support = dev.pipe->get_param(dev.pipe, + PIPE_CAP_RESOURCE_FROM_USER_MEMORY); + + if (image *img = dynamic_cast<image *>(&obj)) { + info.format = translate_format(img->format()); + info.width0 = img->width(); + info.height0 = img->height(); + info.depth0 = img->depth(); + } else { + info.width0 = obj.size(); + info.height0 = 1; + info.depth0 = 1; + } + + info.array_size = 1; + info.target = translate_target(obj.type()); + info.bind = (PIPE_BIND_SAMPLER_VIEW | + PIPE_BIND_COMPUTE_RESOURCE | + PIPE_BIND_GLOBAL); + + if (obj.flags() & CL_MEM_USE_HOST_PTR && user_ptr_support) { + // Page alignment is normally required for this, just try, hope for the + // best and fall back if it fails. + pipe = dev.pipe->resource_from_user_memory(dev.pipe, &info, obj.host_ptr()); + if (pipe) + return; + } + + if (obj.flags() & (CL_MEM_ALLOC_HOST_PTR | CL_MEM_USE_HOST_PTR)) { + info.usage = PIPE_USAGE_STAGING; + } + + pipe = dev.pipe->resource_create(dev.pipe, &info); + if (!pipe) + throw error(CL_OUT_OF_RESOURCES); + + if (obj.flags() & (CL_MEM_USE_HOST_PTR | CL_MEM_COPY_HOST_PTR)) { + const void *data_ptr = !data.empty() ? data.data() : obj.host_ptr(); + box rect { {{ 0, 0, 0 }}, {{ info.width0, info.height0, info.depth0 }} }; + unsigned cpp = util_format_get_blocksize(info.format); + + if (pipe->target == PIPE_BUFFER) + q.pipe->buffer_subdata(q.pipe, pipe, PIPE_TRANSFER_WRITE, + 0, info.width0, data_ptr); + else + q.pipe->texture_subdata(q.pipe, pipe, 0, PIPE_TRANSFER_WRITE, + rect, data_ptr, cpp * info.width0, + cpp * info.width0 * info.height0); + } +} + +root_resource::root_resource(clover::device &dev, memory_obj &obj, + root_resource &r) : + resource(dev, obj) { + assert(0); // XXX -- resource shared among dev and r.dev +} + +root_resource::~root_resource() { + pipe_resource_reference(&this->pipe, NULL); +} + +sub_resource::sub_resource(resource &r, const vector &offset) : + resource(r.device(), r.obj) { + this->pipe = r.pipe; + this->offset = r.offset + offset; +} + +mapping::mapping(command_queue &q, resource &r, + cl_map_flags flags, bool blocking, + const resource::vector &origin, + const resource::vector ®ion) : + pctx(q.pipe), pres(NULL) { + unsigned usage = ((flags & CL_MAP_WRITE ? PIPE_TRANSFER_WRITE : 0 ) | + (flags & CL_MAP_READ ? PIPE_TRANSFER_READ : 0 ) | + (flags & CL_MAP_WRITE_INVALIDATE_REGION ? + PIPE_TRANSFER_DISCARD_RANGE : 0) | + (!blocking ? PIPE_TRANSFER_UNSYNCHRONIZED : 0)); + + p = pctx->transfer_map(pctx, r.pipe, 0, usage, + box(origin + r.offset, region), &pxfer); + if (!p) { + pxfer = NULL; + throw error(CL_OUT_OF_RESOURCES); + } + pipe_resource_reference(&pres, r.pipe); +} + +mapping::mapping(mapping &&m) : + pctx(m.pctx), pxfer(m.pxfer), pres(m.pres), p(m.p) { + m.pctx = NULL; + m.pxfer = NULL; + m.pres = NULL; + m.p = NULL; +} + +mapping::~mapping() { + if (pxfer) { + pctx->transfer_unmap(pctx, pxfer); + } + pipe_resource_reference(&pres, NULL); +} + +mapping & +mapping::operator=(mapping m) { + std::swap(pctx, m.pctx); + std::swap(pxfer, m.pxfer); + std::swap(pres, m.pres); + std::swap(p, m.p); + return *this; +} diff --git a/src/gallium/frontends/clover/core/resource.hpp b/src/gallium/frontends/clover/core/resource.hpp new file mode 100644 index 00000000000..3b994b4008b --- /dev/null +++ b/src/gallium/frontends/clover/core/resource.hpp @@ -0,0 +1,133 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_RESOURCE_HPP +#define CLOVER_CORE_RESOURCE_HPP + +#include <list> + +#include "core/queue.hpp" +#include "util/algebra.hpp" +#include "pipe/p_state.h" + +namespace clover { + class memory_obj; + class mapping; + + /// + /// Class that represents a device-specific instance of some memory + /// object. + /// + class resource { + public: + typedef std::array<size_t, 3> vector; + + virtual ~resource(); + + resource(const resource &r) = delete; + resource & + operator=(const resource &r) = delete; + + void copy(command_queue &q, const vector &origin, const vector ®ion, + resource &src_resource, const vector &src_origin); + + void *add_map(command_queue &q, cl_map_flags flags, bool blocking, + const vector &origin, const vector ®ion); + void del_map(void *p); + unsigned map_count() const; + + const intrusive_ref<clover::device> device; + memory_obj &obj; + + friend class sub_resource; + friend class mapping; + friend class kernel; + + protected: + resource(clover::device &dev, memory_obj &obj); + + pipe_sampler_view *bind_sampler_view(command_queue &q); + void unbind_sampler_view(command_queue &q, + pipe_sampler_view *st); + + pipe_surface *bind_surface(command_queue &q, bool rw); + void unbind_surface(command_queue &q, pipe_surface *st); + + pipe_resource *pipe; + vector offset; + + private: + std::list<mapping> maps; + }; + + /// + /// Resource associated with its own top-level data storage + /// allocated in some device. + /// + class root_resource : public resource { + public: + root_resource(clover::device &dev, memory_obj &obj, + command_queue &q, const std::string &data); + root_resource(clover::device &dev, memory_obj &obj, root_resource &r); + virtual ~root_resource(); + }; + + /// + /// Resource that reuses a portion of some other resource as data + /// storage. + /// + class sub_resource : public resource { + public: + sub_resource(resource &r, const vector &offset); + }; + + /// + /// Class that represents a mapping of some resource into the CPU + /// memory space. + /// + class mapping { + public: + mapping(command_queue &q, resource &r, cl_map_flags flags, + bool blocking, const resource::vector &origin, + const resource::vector ®ion); + mapping(mapping &&m); + ~mapping(); + + mapping & + operator=(mapping m); + + mapping(const mapping &m) = delete; + + template<typename T> + operator T *() const { + return (T *)p; + } + + private: + pipe_context *pctx; + pipe_transfer *pxfer; + pipe_resource *pres; + void *p; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/sampler.cpp b/src/gallium/frontends/clover/core/sampler.cpp new file mode 100644 index 00000000000..6f2784b538e --- /dev/null +++ b/src/gallium/frontends/clover/core/sampler.cpp @@ -0,0 +1,73 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/sampler.hpp" +#include "pipe/p_state.h" + +using namespace clover; + +sampler::sampler(clover::context &ctx, bool norm_mode, + cl_addressing_mode addr_mode, + cl_filter_mode filter_mode) : + context(ctx), _norm_mode(norm_mode), + _addr_mode(addr_mode), _filter_mode(filter_mode) { +} + +bool +sampler::norm_mode() { + return _norm_mode; +} + +cl_addressing_mode +sampler::addr_mode() { + return _addr_mode; +} + +cl_filter_mode +sampler::filter_mode() { + return _filter_mode; +} + +void * +sampler::bind(command_queue &q) { + struct pipe_sampler_state info {}; + + info.normalized_coords = norm_mode(); + + info.wrap_s = info.wrap_t = info.wrap_r = + (addr_mode() == CL_ADDRESS_CLAMP_TO_EDGE ? PIPE_TEX_WRAP_CLAMP_TO_EDGE : + addr_mode() == CL_ADDRESS_CLAMP ? PIPE_TEX_WRAP_CLAMP_TO_BORDER : + addr_mode() == CL_ADDRESS_REPEAT ? PIPE_TEX_WRAP_REPEAT : + addr_mode() == CL_ADDRESS_MIRRORED_REPEAT ? PIPE_TEX_WRAP_MIRROR_REPEAT : + PIPE_TEX_WRAP_CLAMP_TO_EDGE); + + info.min_img_filter = info.mag_img_filter = + (filter_mode() == CL_FILTER_LINEAR ? PIPE_TEX_FILTER_LINEAR : + PIPE_TEX_FILTER_NEAREST); + + return q.pipe->create_sampler_state(q.pipe, &info); +} + +void +sampler::unbind(command_queue &q, void *st) { + q.pipe->delete_sampler_state(q.pipe, st); +} diff --git a/src/gallium/frontends/clover/core/sampler.hpp b/src/gallium/frontends/clover/core/sampler.hpp new file mode 100644 index 00000000000..2632c3067fa --- /dev/null +++ b/src/gallium/frontends/clover/core/sampler.hpp @@ -0,0 +1,58 @@ +// +// Copyright 2012 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_SAMPLER_HPP +#define CLOVER_CORE_SAMPLER_HPP + +#include "core/object.hpp" +#include "core/queue.hpp" + +namespace clover { + class sampler : public ref_counter, public _cl_sampler { + public: + sampler(clover::context &ctx, bool norm_mode, + cl_addressing_mode addr_mode, + cl_filter_mode filter_mode); + + sampler(const sampler &s) = delete; + sampler & + operator=(const sampler &s) = delete; + + bool norm_mode(); + cl_addressing_mode addr_mode(); + cl_filter_mode filter_mode(); + + const intrusive_ref<clover::context> context; + + friend class kernel; + + private: + void *bind(command_queue &q); + void unbind(command_queue &q, void *st); + + bool _norm_mode; + cl_addressing_mode _addr_mode; + cl_filter_mode _filter_mode; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/core/timestamp.cpp b/src/gallium/frontends/clover/core/timestamp.cpp new file mode 100644 index 00000000000..3fd341f30da --- /dev/null +++ b/src/gallium/frontends/clover/core/timestamp.cpp @@ -0,0 +1,64 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "core/timestamp.hpp" +#include "core/queue.hpp" +#include "pipe/p_screen.h" +#include "pipe/p_context.h" + +using namespace clover; + +timestamp::query::query(command_queue &q) : + q(q), + _query(q.pipe->create_query(q.pipe, PIPE_QUERY_TIMESTAMP, 0)) { + q.pipe->end_query(q.pipe, _query); +} + +timestamp::query::query(query &&other) : + q(other.q), + _query(other._query) { + other._query = NULL; +} + +timestamp::query::~query() { + if (_query) + q().pipe->destroy_query(q().pipe, _query); +} + +cl_ulong +timestamp::query::operator()() const { + pipe_query_result result; + + if (!q().pipe->get_query_result(q().pipe, _query, false, &result)) + throw error(CL_PROFILING_INFO_NOT_AVAILABLE); + + return result.u64; +} + +timestamp::current::current(command_queue &q) : + result(q.pipe->screen->get_timestamp(q.pipe->screen)) { +} + +cl_ulong +timestamp::current::operator()() const { + return result; +} diff --git a/src/gallium/frontends/clover/core/timestamp.hpp b/src/gallium/frontends/clover/core/timestamp.hpp new file mode 100644 index 00000000000..b4b2c83eb92 --- /dev/null +++ b/src/gallium/frontends/clover/core/timestamp.hpp @@ -0,0 +1,74 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_CORE_TIMESTAMP_HPP +#define CLOVER_CORE_TIMESTAMP_HPP + +#include "core/object.hpp" + +struct pipe_query; + +namespace clover { + class command_queue; + + namespace timestamp { + /// + /// Emit a timestamp query that is executed asynchronously by + /// the command queue \a q. + /// + class query { + public: + query(command_queue &q); + query(query &&other); + ~query(); + + query &operator=(const query &) = delete; + + /// + /// Retrieve the query results. + /// + cl_ulong operator()() const; + + private: + const intrusive_ref<command_queue> q; + pipe_query *_query; + }; + + /// + /// Get the current timestamp value. + /// + class current { + public: + current(command_queue &q); + + /// + /// Retrieve the query results. + /// + cl_ulong operator()() const; + + private: + cl_ulong result; + }; + } +} + +#endif diff --git a/src/gallium/frontends/clover/llvm/codegen.hpp b/src/gallium/frontends/clover/llvm/codegen.hpp new file mode 100644 index 00000000000..e35627c4729 --- /dev/null +++ b/src/gallium/frontends/clover/llvm/codegen.hpp @@ -0,0 +1,68 @@ +// +// Copyright 2016 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +/// +/// \file +/// Tools to generate various forms of binary code from existing LLVM IR in +/// the given llvm::Module object and output the result as a clover::module. +/// + +#ifndef CLOVER_LLVM_CODEGEN_HPP +#define CLOVER_LLVM_CODEGEN_HPP + +#include "llvm/util.hpp" +#include "core/module.hpp" + +#include <llvm/IR/Module.h> + +#include <clang/Frontend/CompilerInstance.h> + +namespace clover { + namespace llvm { + std::string + print_module_bitcode(const ::llvm::Module &mod); + + module + build_module_library(const ::llvm::Module &mod, + enum module::section::type section_type); + + std::unique_ptr< ::llvm::Module> + parse_module_library(const module &m, ::llvm::LLVMContext &ctx, + std::string &r_log); + + module + build_module_native(::llvm::Module &mod, const target &target, + const clang::CompilerInstance &c, + std::string &r_log); + + std::string + print_module_native(const ::llvm::Module &mod, const target &target); + + module + build_module_common(const ::llvm::Module &mod, + const std::vector<char> &code, + const std::map<std::string, unsigned> &offsets, + const clang::CompilerInstance &c); + } +} + +#endif diff --git a/src/gallium/frontends/clover/llvm/codegen/bitcode.cpp b/src/gallium/frontends/clover/llvm/codegen/bitcode.cpp new file mode 100644 index 00000000000..7434e8cf6c9 --- /dev/null +++ b/src/gallium/frontends/clover/llvm/codegen/bitcode.cpp @@ -0,0 +1,91 @@ +// +// Copyright 2012-2016 Francisco Jerez +// Copyright 2012-2016 Advanced Micro Devices, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +/// +/// \file +/// Trivial codegen back-end that simply passes through the existing LLVM IR +/// and either formats it so it can be consumed by pipe drivers (if +/// build_module_bitcode() is used) or serializes so it can be deserialized at +/// a later point and passed to the actual codegen back-end (if +/// build_module_library() / parse_module_library() is used), potentially +/// after linking against other bitcode object files. +/// + +#include "llvm/codegen.hpp" +#include "llvm/compat.hpp" +#include "llvm/metadata.hpp" +#include "core/error.hpp" +#include "util/algorithm.hpp" + +#include <map> +#include <llvm/Config/llvm-config.h> +#if LLVM_VERSION_MAJOR < 4 +#include <llvm/Bitcode/ReaderWriter.h> +#else +#include <llvm/Bitcode/BitcodeReader.h> +#include <llvm/Bitcode/BitcodeWriter.h> +#endif +#include <llvm/Support/raw_ostream.h> + +using namespace clover; +using namespace clover::llvm; + +namespace { + std::vector<char> + emit_code(const ::llvm::Module &mod) { + ::llvm::SmallVector<char, 1024> data; + ::llvm::raw_svector_ostream os { data }; + compat::write_bitcode_to_file(mod, os); + return { os.str().begin(), os.str().end() }; + } +} + +std::string +clover::llvm::print_module_bitcode(const ::llvm::Module &mod) { + std::string s; + ::llvm::raw_string_ostream os { s }; + mod.print(os, NULL); + return os.str(); +} + +module +clover::llvm::build_module_library(const ::llvm::Module &mod, + enum module::section::type section_type) { + module m; + const auto code = emit_code(mod); + m.secs.emplace_back(0, section_type, code.size(), code); + return m; +} + +std::unique_ptr< ::llvm::Module> +clover::llvm::parse_module_library(const module &m, ::llvm::LLVMContext &ctx, + std::string &r_log) { + auto mod = ::llvm::parseBitcodeFile(::llvm::MemoryBufferRef( + as_string(m.secs[0].data), " "), ctx); + + compat::handle_module_error(mod, [&](const std::string &s) { + fail(r_log, error(CL_INVALID_PROGRAM), s); + }); + + return std::unique_ptr< ::llvm::Module>(std::move(*mod)); +} diff --git a/src/gallium/frontends/clover/llvm/codegen/common.cpp b/src/gallium/frontends/clover/llvm/codegen/common.cpp new file mode 100644 index 00000000000..730ba5a2ff7 --- /dev/null +++ b/src/gallium/frontends/clover/llvm/codegen/common.cpp @@ -0,0 +1,209 @@ +// +// Copyright 2012-2016 Francisco Jerez +// Copyright 2012-2016 Advanced Micro Devices, Inc. +// Copyright 2015 Zoltan Gilian +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +/// +/// \file +/// Codegen back-end-independent part of the construction of an executable +/// clover::module, including kernel argument metadata extraction and +/// formatting of the pre-generated binary code in a form that can be +/// understood by pipe drivers. +/// + +#include "llvm/codegen.hpp" +#include "llvm/metadata.hpp" + +#include "CL/cl.h" + +#include "pipe/p_state.h" +#include "util/u_math.h" + +#include <clang/Basic/TargetInfo.h> + +using namespace clover; +using namespace clover::llvm; + +using ::llvm::Module; +using ::llvm::Function; +using ::llvm::Type; +using ::llvm::isa; +using ::llvm::cast; +using ::llvm::dyn_cast; + +namespace { + enum module::argument::type + get_image_type(const std::string &type, + const std::string &qual) { + if (type == "image2d_t" && qual == "read_only") + return module::argument::image2d_rd; + else if (type == "image2d_t" && qual == "write_only") + return module::argument::image2d_wr; + else if (type == "image3d_t" && qual == "read_only") + return module::argument::image3d_rd; + else if (type == "image3d_t" && qual == "write_only") + return module::argument::image3d_wr; + else + unreachable("Unknown image type"); + } + + std::vector<module::argument> + make_kernel_args(const Module &mod, const std::string &kernel_name, + const clang::CompilerInstance &c) { + std::vector<module::argument> args; + const Function &f = *mod.getFunction(kernel_name); + ::llvm::DataLayout dl(&mod); + const auto size_type = + dl.getSmallestLegalIntType(mod.getContext(), sizeof(cl_uint) * 8); + + for (const auto &arg : f.args()) { + const auto arg_type = arg.getType(); + + // OpenCL 1.2 specification, Ch. 6.1.5: "A built-in data + // type that is not a power of two bytes in size must be + // aligned to the next larger power of two. + // This rule applies to built-in types only, not structs or unions." + const unsigned arg_store_size = dl.getTypeStoreSize(arg_type); + const unsigned arg_api_size = dl.getTypeAllocSize(arg_type); + + const auto target_type = compat::get_abi_type(arg_type, mod); + const unsigned target_size = dl.getTypeStoreSize(target_type); + const unsigned target_align = dl.getABITypeAlignment(target_type); + + const auto type_name = get_argument_metadata(f, arg, + "kernel_arg_type"); + + if (type_name == "image2d_t" || type_name == "image3d_t") { + // Image. + const auto access_qual = get_argument_metadata( + f, arg, "kernel_arg_access_qual"); + args.emplace_back(get_image_type(type_name, access_qual), + arg_store_size, target_size, + target_align, module::argument::zero_ext); + + } else if (type_name == "__llvm_image_size") { + // Image size implicit argument. + args.emplace_back(module::argument::scalar, sizeof(cl_uint), + dl.getTypeStoreSize(size_type), + dl.getABITypeAlignment(size_type), + module::argument::zero_ext, + module::argument::image_size); + + } else if (type_name == "__llvm_image_format") { + // Image format implicit argument. + args.emplace_back(module::argument::scalar, sizeof(cl_uint), + dl.getTypeStoreSize(size_type), + dl.getABITypeAlignment(size_type), + module::argument::zero_ext, + module::argument::image_format); + + } else { + // Other types. + const auto actual_type = + isa< ::llvm::PointerType>(arg_type) && arg.hasByValAttr() ? + cast< ::llvm::PointerType>(arg_type)->getElementType() : arg_type; + + if (actual_type->isPointerTy()) { + const unsigned address_space = + cast< ::llvm::PointerType>(actual_type)->getAddressSpace(); + + if (address_space == compat::target_address_space( + c.getTarget(), clang::LangAS::opencl_local)) { + args.emplace_back(module::argument::local, arg_api_size, + target_size, target_align, + module::argument::zero_ext); + } else { + // XXX: Correctly handle constant address space. There is no + // way for r600g to pass a handle for constant buffers back + // to clover like it can for global buffers, so + // creating constant arguments will break r600g. For now, + // continue treating constant buffers as global buffers + // until we can come up with a way to create handles for + // constant buffers. + args.emplace_back(module::argument::global, arg_api_size, + target_size, target_align, + module::argument::zero_ext); + } + + } else { + const bool needs_sign_ext = f.getAttributes().hasAttribute( + arg.getArgNo() + 1, ::llvm::Attribute::SExt); + + args.emplace_back(module::argument::scalar, arg_api_size, + target_size, target_align, + (needs_sign_ext ? module::argument::sign_ext : + module::argument::zero_ext)); + } + } + } + + // Append implicit arguments. XXX - The types, ordering and + // vector size of the implicit arguments should depend on the + // target according to the selected calling convention. + args.emplace_back(module::argument::scalar, sizeof(cl_uint), + dl.getTypeStoreSize(size_type), + dl.getABITypeAlignment(size_type), + module::argument::zero_ext, + module::argument::grid_dimension); + + args.emplace_back(module::argument::scalar, sizeof(cl_uint), + dl.getTypeStoreSize(size_type), + dl.getABITypeAlignment(size_type), + module::argument::zero_ext, + module::argument::grid_offset); + + return args; + } + + module::section + make_text_section(const std::vector<char> &code) { + const pipe_binary_program_header header { uint32_t(code.size()) }; + module::section text { 0, module::section::text_executable, + header.num_bytes, {} }; + + text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header), + reinterpret_cast<const char *>(&header) + sizeof(header)); + text.data.insert(text.data.end(), code.begin(), code.end()); + + return text; + } +} + +module +clover::llvm::build_module_common(const Module &mod, + const std::vector<char> &code, + const std::map<std::string, + unsigned> &offsets, + const clang::CompilerInstance &c) { + module m; + + for (const auto &llvm_name : map(std::mem_fn(&Function::getName), + get_kernels(mod))) { + const ::std::string name(llvm_name); + if (offsets.count(name)) + m.syms.emplace_back(name, 0, offsets.at(name), + make_kernel_args(mod, name, c)); + } + + m.secs.push_back(make_text_section(code)); + return m; +} diff --git a/src/gallium/frontends/clover/llvm/codegen/native.cpp b/src/gallium/frontends/clover/llvm/codegen/native.cpp new file mode 100644 index 00000000000..52346131200 --- /dev/null +++ b/src/gallium/frontends/clover/llvm/codegen/native.cpp @@ -0,0 +1,163 @@ +// +// Copyright 2012-2016 Francisco Jerez +// Copyright 2012-2016 Advanced Micro Devices, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +/// +/// \file +/// Generate code using an arbitrary LLVM back-end capable of emitting +/// executable code as an ELF object file. +/// + +#include "llvm/codegen.hpp" +#include "llvm/compat.hpp" +#include "llvm/util.hpp" +#include "core/error.hpp" + +#include <llvm/Target/TargetMachine.h> +#include <llvm/Support/TargetRegistry.h> +#include <llvm/Transforms/Utils/Cloning.h> + +#include <libelf.h> +#include <gelf.h> + +using namespace clover; +using namespace clover::llvm; +using ::llvm::TargetMachine; + +namespace { + namespace elf { + std::unique_ptr<Elf, int (*)(Elf *)> + get(const std::vector<char> &code) { + // One of the libelf implementations + // (http://www.mr511.de/software/english.htm) requires calling + // elf_version() before elf_memory(). + elf_version(EV_CURRENT); + return { elf_memory(const_cast<char *>(code.data()), code.size()), + elf_end }; + } + + Elf_Scn * + get_symbol_table(Elf *elf) { + size_t section_str_index; + elf_getshdrstrndx(elf, §ion_str_index); + + for (Elf_Scn *s = elf_nextscn(elf, NULL); s; s = elf_nextscn(elf, s)) { + GElf_Shdr header; + if (gelf_getshdr(s, &header) != &header) + return nullptr; + + if (!std::strcmp(elf_strptr(elf, section_str_index, header.sh_name), + ".symtab")) + return s; + } + + return nullptr; + } + + std::map<std::string, unsigned> + get_symbol_offsets(Elf *elf, Elf_Scn *symtab) { + Elf_Data *const symtab_data = elf_getdata(symtab, NULL); + GElf_Shdr header; + if (gelf_getshdr(symtab, &header) != &header) + return {}; + + std::map<std::string, unsigned> symbol_offsets; + GElf_Sym symbol; + unsigned i = 0; + + while (GElf_Sym *s = gelf_getsym(symtab_data, i++, &symbol)) { + const char *name = elf_strptr(elf, header.sh_link, s->st_name); + symbol_offsets[name] = s->st_value; + } + + return symbol_offsets; + } + } + + std::map<std::string, unsigned> + get_symbol_offsets(const std::vector<char> &code, std::string &r_log) { + const auto elf = elf::get(code); + const auto symtab = elf::get_symbol_table(elf.get()); + if (!symtab) + fail(r_log, build_error(), "Unable to find symbol table."); + + return elf::get_symbol_offsets(elf.get(), symtab); + } + + std::vector<char> + emit_code(::llvm::Module &mod, const target &target, + compat::CodeGenFileType ft, + std::string &r_log) { + std::string err; + auto t = ::llvm::TargetRegistry::lookupTarget(target.triple, err); + if (!t) + fail(r_log, build_error(), err); + + std::unique_ptr<TargetMachine> tm { + t->createTargetMachine(target.triple, target.cpu, "", {}, + ::llvm::None, compat::default_code_model, + ::llvm::CodeGenOpt::Default) }; + if (!tm) + fail(r_log, build_error(), + "Could not create TargetMachine: " + target.triple); + + ::llvm::SmallVector<char, 1024> data; + + { + ::llvm::legacy::PassManager pm; + ::llvm::raw_svector_ostream os { data }; + + mod.setDataLayout(tm->createDataLayout()); + tm->Options.MCOptions.AsmVerbose = + (ft == compat::CGFT_AssemblyFile); + + if (compat::add_passes_to_emit_file(*tm, pm, os, ft)) + fail(r_log, build_error(), "TargetMachine can't emit this file"); + + pm.run(mod); + } + + return { data.begin(), data.end() }; + } +} + +module +clover::llvm::build_module_native(::llvm::Module &mod, const target &target, + const clang::CompilerInstance &c, + std::string &r_log) { + const auto code = emit_code(mod, target, + compat::CGFT_ObjectFile, r_log); + return build_module_common(mod, code, get_symbol_offsets(code, r_log), c); +} + +std::string +clover::llvm::print_module_native(const ::llvm::Module &mod, + const target &target) { + std::string log; + try { + std::unique_ptr< ::llvm::Module> cmod { compat::clone_module(mod) }; + return as_string(emit_code(*cmod, target, + compat::CGFT_AssemblyFile, log)); + } catch (...) { + return "Couldn't output native disassembly: " + log; + } +} diff --git a/src/gallium/frontends/clover/llvm/compat.hpp b/src/gallium/frontends/clover/llvm/compat.hpp new file mode 100644 index 00000000000..51902739acc --- /dev/null +++ b/src/gallium/frontends/clover/llvm/compat.hpp @@ -0,0 +1,216 @@ +// +// Copyright 2016 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +/// +/// \file +/// Some thin wrappers around the Clang/LLVM API used to preserve +/// compatibility with older API versions while keeping the ifdef clutter low +/// in the rest of the clover::llvm subtree. In case of an API break please +/// consider whether it's possible to preserve backwards compatibility by +/// introducing a new one-liner inline function or typedef here under the +/// compat namespace in order to keep the running code free from preprocessor +/// conditionals. +/// + +#ifndef CLOVER_LLVM_COMPAT_HPP +#define CLOVER_LLVM_COMPAT_HPP + +#include "util/algorithm.hpp" + +#include <llvm/Config/llvm-config.h> +#if LLVM_VERSION_MAJOR < 4 +#include <llvm/Bitcode/ReaderWriter.h> +#else +#include <llvm/Bitcode/BitcodeReader.h> +#include <llvm/Bitcode/BitcodeWriter.h> +#endif + +#include <llvm/IR/LLVMContext.h> +#include <llvm/Linker/Linker.h> +#include <llvm/Transforms/IPO.h> +#include <llvm/Transforms/Utils/Cloning.h> +#include <llvm/Target/TargetMachine.h> +#if LLVM_VERSION_MAJOR >= 4 +#include <llvm/Support/Error.h> +#else +#include <llvm/Support/ErrorOr.h> +#endif + +#include <llvm/IR/LegacyPassManager.h> +#include <llvm/Analysis/TargetLibraryInfo.h> + +#include <clang/Basic/TargetInfo.h> +#include <clang/Frontend/CompilerInstance.h> + +#if LLVM_VERSION_MAJOR >= 8 +#include <clang/Basic/CodeGenOptions.h> +#else +#include <clang/Frontend/CodeGenOptions.h> +#endif + +#if LLVM_VERSION_MAJOR >= 10 +#include <llvm/Support/CodeGen.h> +#endif + +namespace clover { + namespace llvm { + namespace compat { + +#if LLVM_VERSION_MAJOR >= 10 + const auto CGFT_ObjectFile = ::llvm::CGFT_ObjectFile; + const auto CGFT_AssemblyFile = ::llvm::CGFT_AssemblyFile; + typedef ::llvm::CodeGenFileType CodeGenFileType; +#else + const auto CGFT_ObjectFile = ::llvm::TargetMachine::CGFT_ObjectFile; + const auto CGFT_AssemblyFile = + ::llvm::TargetMachine::CGFT_AssemblyFile; + typedef ::llvm::TargetMachine::CodeGenFileType CodeGenFileType; +#endif + + template<typename T, typename AS> + unsigned target_address_space(const T &target, const AS lang_as) { + const auto &map = target.getAddressSpaceMap(); +#if LLVM_VERSION_MAJOR >= 5 + return map[static_cast<unsigned>(lang_as)]; +#else + return map[lang_as - clang::LangAS::Offset]; +#endif + } + +#if LLVM_VERSION_MAJOR >= 10 + const clang::InputKind ik_opencl = clang::Language::OpenCL; +#elif LLVM_VERSION_MAJOR >= 5 + const clang::InputKind ik_opencl = clang::InputKind::OpenCL; +#else + const clang::InputKind ik_opencl = clang::IK_OpenCL; +#endif + +#if LLVM_VERSION_MAJOR >= 5 + const clang::LangStandard::Kind lang_opencl10 = clang::LangStandard::lang_opencl10; +#else + const clang::LangStandard::Kind lang_opencl10 = clang::LangStandard::lang_opencl; +#endif + + inline void + add_link_bitcode_file(clang::CodeGenOptions &opts, + const std::string &path) { +#if LLVM_VERSION_MAJOR >= 5 + clang::CodeGenOptions::BitcodeFileToLink F; + + F.Filename = path; + F.PropagateAttrs = true; + F.LinkFlags = ::llvm::Linker::Flags::None; + opts.LinkBitcodeFiles.emplace_back(F); +#else + opts.LinkBitcodeFiles.emplace_back(::llvm::Linker::Flags::None, path); +#endif + } + +#if LLVM_VERSION_MAJOR >= 6 + const auto default_code_model = ::llvm::None; +#else + const auto default_code_model = ::llvm::CodeModel::Default; +#endif + + template<typename M, typename F> void + handle_module_error(M &mod, const F &f) { +#if LLVM_VERSION_MAJOR >= 4 + if (::llvm::Error err = mod.takeError()) + ::llvm::handleAllErrors(std::move(err), [&](::llvm::ErrorInfoBase &eib) { + f(eib.message()); + }); +#else + if (!mod) + f(mod.getError().message()); +#endif + } + + template<typename T> void + set_diagnostic_handler(::llvm::LLVMContext &ctx, + T *diagnostic_handler, void *data) { +#if LLVM_VERSION_MAJOR >= 6 + ctx.setDiagnosticHandlerCallBack(diagnostic_handler, data); +#else + ctx.setDiagnosticHandler(diagnostic_handler, data); +#endif + } + + inline std::unique_ptr< ::llvm::Module> + clone_module(const ::llvm::Module &mod) + { +#if LLVM_VERSION_MAJOR >= 7 + return ::llvm::CloneModule(mod); +#else + return ::llvm::CloneModule(&mod); +#endif + } + + template<typename T> void + write_bitcode_to_file(const ::llvm::Module &mod, T &os) + { +#if LLVM_VERSION_MAJOR >= 7 + ::llvm::WriteBitcodeToFile(mod, os); +#else + ::llvm::WriteBitcodeToFile(&mod, os); +#endif + } + + template<typename TM, typename PM, typename OS, typename FT> + bool add_passes_to_emit_file(TM &tm, PM &pm, OS &os, FT &ft) + { +#if LLVM_VERSION_MAJOR >= 7 + return tm.addPassesToEmitFile(pm, os, nullptr, ft); +#else + return tm.addPassesToEmitFile(pm, os, ft); +#endif + } + + template<typename T> inline bool + create_compiler_invocation_from_args(clang::CompilerInvocation &cinv, + T copts, + clang::DiagnosticsEngine &diag) + { +#if LLVM_VERSION_MAJOR >= 10 + return clang::CompilerInvocation::CreateFromArgs( + cinv, copts, diag); +#else + return clang::CompilerInvocation::CreateFromArgs( + cinv, copts.data(), copts.data() + copts.size(), diag); +#endif + } + + template<typename T, typename M> + T get_abi_type(const T &arg_type, const M &mod) { +#if LLVM_VERSION_MAJOR >= 7 + return arg_type; +#else + ::llvm::DataLayout dl(&mod); + const unsigned arg_store_size = dl.getTypeStoreSize(arg_type); + return !arg_type->isIntegerTy() ? arg_type : + dl.getSmallestLegalIntType(mod.getContext(), arg_store_size * 8); +#endif + } + } + } +} + +#endif diff --git a/src/gallium/frontends/clover/llvm/invocation.cpp b/src/gallium/frontends/clover/llvm/invocation.cpp new file mode 100644 index 00000000000..95a9d036622 --- /dev/null +++ b/src/gallium/frontends/clover/llvm/invocation.cpp @@ -0,0 +1,465 @@ +// +// Copyright 2012-2016 Francisco Jerez +// Copyright 2012-2016 Advanced Micro Devices, Inc. +// Copyright 2014-2016 Jan Vesely +// Copyright 2014-2015 Serge Martin +// Copyright 2015 Zoltan Gilian +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include <llvm/IR/DiagnosticPrinter.h> +#include <llvm/IR/DiagnosticInfo.h> +#include <llvm/IR/LLVMContext.h> +#include <llvm/Support/raw_ostream.h> +#include <llvm/Transforms/IPO/PassManagerBuilder.h> +#include <llvm-c/Target.h> +#ifdef HAVE_CLOVER_SPIRV +#include <LLVMSPIRVLib/LLVMSPIRVLib.h> +#endif + +#include <clang/CodeGen/CodeGenAction.h> +#include <clang/Lex/PreprocessorOptions.h> +#include <clang/Frontend/TextDiagnosticBuffer.h> +#include <clang/Frontend/TextDiagnosticPrinter.h> +#include <clang/Basic/TargetInfo.h> + +// We need to include internal headers last, because the internal headers +// include CL headers which have #define's like: +// +//#define cl_khr_gl_sharing 1 +//#define cl_khr_icd 1 +// +// Which will break the compilation of clang/Basic/OpenCLOptions.h + +#include "core/error.hpp" +#include "llvm/codegen.hpp" +#include "llvm/compat.hpp" +#include "llvm/invocation.hpp" +#include "llvm/metadata.hpp" +#include "llvm/util.hpp" +#ifdef HAVE_CLOVER_SPIRV +#include "spirv/invocation.hpp" +#endif +#include "util/algorithm.hpp" + + +using namespace clover; +using namespace clover::llvm; + +using ::llvm::Function; +using ::llvm::LLVMContext; +using ::llvm::Module; +using ::llvm::raw_string_ostream; + +namespace { + + struct cl_version { + std::string version_str; // CL Version + unsigned version_number; // Numeric CL Version + }; + + static const unsigned ANY_VERSION = 999; + const cl_version cl_versions[] = { + { "1.0", 100}, + { "1.1", 110}, + { "1.2", 120}, + { "2.0", 200}, + { "2.1", 210}, + { "2.2", 220}, + }; + + struct clc_version_lang_std { + unsigned version_number; // CLC Version + clang::LangStandard::Kind clc_lang_standard; + }; + + const clc_version_lang_std cl_version_lang_stds[] = { + { 100, compat::lang_opencl10}, + { 110, clang::LangStandard::lang_opencl11}, + { 120, clang::LangStandard::lang_opencl12}, + { 200, clang::LangStandard::lang_opencl20}, + }; + + void + init_targets() { + static bool targets_initialized = false; + if (!targets_initialized) { + LLVMInitializeAllTargets(); + LLVMInitializeAllTargetInfos(); + LLVMInitializeAllTargetMCs(); + LLVMInitializeAllAsmParsers(); + LLVMInitializeAllAsmPrinters(); + targets_initialized = true; + } + } + + void + diagnostic_handler(const ::llvm::DiagnosticInfo &di, void *data) { + if (di.getSeverity() == ::llvm::DS_Error) { + raw_string_ostream os { *reinterpret_cast<std::string *>(data) }; + ::llvm::DiagnosticPrinterRawOStream printer { os }; + di.print(printer); + throw build_error(); + } + } + + std::unique_ptr<LLVMContext> + create_context(std::string &r_log) { + init_targets(); + std::unique_ptr<LLVMContext> ctx { new LLVMContext }; + compat::set_diagnostic_handler(*ctx, diagnostic_handler, &r_log); + return ctx; + } + + const struct clc_version_lang_std& + get_cl_lang_standard(unsigned requested, unsigned max = ANY_VERSION) { + for (const struct clc_version_lang_std &version : cl_version_lang_stds) { + if (version.version_number == max || + version.version_number == requested) { + return version; + } + } + throw build_error("Unknown/Unsupported language version"); + } + + const struct cl_version& + get_cl_version(const std::string &version_str, + unsigned max = ANY_VERSION) { + for (const struct cl_version &version : cl_versions) { + if (version.version_number == max || version.version_str == version_str) { + return version; + } + } + throw build_error("Unknown/Unsupported language version"); + } + + clang::LangStandard::Kind + get_lang_standard_from_version_str(const std::string &version_str, + bool is_build_opt = false) { + + //Per CL 2.0 spec, section 5.8.4.5: + // If it's an option, use the value directly. + // If it's a device version, clamp to max 1.x version, a.k.a. 1.2 + const cl_version version = + get_cl_version(version_str, is_build_opt ? ANY_VERSION : 120); + + const struct clc_version_lang_std standard = + get_cl_lang_standard(version.version_number); + + return standard.clc_lang_standard; + } + + clang::LangStandard::Kind + get_language_version(const std::vector<std::string> &opts, + const std::string &device_version) { + + const std::string search = "-cl-std=CL"; + + for (auto &opt: opts) { + auto pos = opt.find(search); + if (pos == 0){ + const auto ver = opt.substr(pos + search.size()); + const auto device_ver = get_cl_version(device_version); + const auto requested = get_cl_version(ver); + if (requested.version_number > device_ver.version_number) { + throw build_error(); + } + return get_lang_standard_from_version_str(ver, true); + } + } + + return get_lang_standard_from_version_str(device_version); + } + + std::unique_ptr<clang::CompilerInstance> + create_compiler_instance(const device &dev, const std::string& ir_target, + const std::vector<std::string> &opts, + std::string &r_log) { + std::unique_ptr<clang::CompilerInstance> c { new clang::CompilerInstance }; + clang::TextDiagnosticBuffer *diag_buffer = new clang::TextDiagnosticBuffer; + clang::DiagnosticsEngine diag { new clang::DiagnosticIDs, + new clang::DiagnosticOptions, diag_buffer }; + + // Parse the compiler options. A file name should be present at the end + // and must have the .cl extension in order for the CompilerInvocation + // class to recognize it as an OpenCL source file. + const std::vector<const char *> copts = + map(std::mem_fn(&std::string::c_str), opts); + + const target &target = ir_target; + const std::string &device_clc_version = dev.device_clc_version(); + + if (!compat::create_compiler_invocation_from_args( + c->getInvocation(), copts, diag)) + throw invalid_build_options_error(); + + diag_buffer->FlushDiagnostics(diag); + if (diag.hasErrorOccurred()) + throw invalid_build_options_error(); + + c->getTargetOpts().CPU = target.cpu; + c->getTargetOpts().Triple = target.triple; + c->getLangOpts().NoBuiltin = true; + + // This is a workaround for a Clang bug which causes the number + // of warnings and errors to be printed to stderr. + // http://www.llvm.org/bugs/show_bug.cgi?id=19735 + c->getDiagnosticOpts().ShowCarets = false; + + c->getInvocation().setLangDefaults(c->getLangOpts(), + compat::ik_opencl, ::llvm::Triple(target.triple), + c->getPreprocessorOpts(), + get_language_version(opts, device_clc_version)); + + c->createDiagnostics(new clang::TextDiagnosticPrinter( + *new raw_string_ostream(r_log), + &c->getDiagnosticOpts(), true)); + + c->setTarget(clang::TargetInfo::CreateTargetInfo( + c->getDiagnostics(), c->getInvocation().TargetOpts)); + + return c; + } + + std::unique_ptr<Module> + compile(LLVMContext &ctx, clang::CompilerInstance &c, + const std::string &name, const std::string &source, + const header_map &headers, const device &dev, + const std::string &opts, bool use_libclc, std::string &r_log) { + c.getFrontendOpts().ProgramAction = clang::frontend::EmitLLVMOnly; + c.getHeaderSearchOpts().UseBuiltinIncludes = true; + c.getHeaderSearchOpts().UseStandardSystemIncludes = true; + c.getHeaderSearchOpts().ResourceDir = CLANG_RESOURCE_DIR; + + if (use_libclc) { + // Add libclc generic search path + c.getHeaderSearchOpts().AddPath(LIBCLC_INCLUDEDIR, + clang::frontend::Angled, + false, false); + + // Add libclc include + c.getPreprocessorOpts().Includes.push_back("clc/clc.h"); + } else { + // Add opencl-c generic search path + c.getHeaderSearchOpts().AddPath(CLANG_RESOURCE_DIR, + clang::frontend::Angled, + false, false); + + // Add opencl include + c.getPreprocessorOpts().Includes.push_back("opencl-c.h"); + } + + // Add definition for the OpenCL version + c.getPreprocessorOpts().addMacroDef("__OPENCL_VERSION__=" + + std::to_string(get_cl_version( + dev.device_version()).version_number)); + + // clc.h requires that this macro be defined: + c.getPreprocessorOpts().addMacroDef("cl_clang_storage_class_specifiers"); + c.getPreprocessorOpts().addRemappedFile( + name, ::llvm::MemoryBuffer::getMemBuffer(source).release()); + + if (headers.size()) { + const std::string tmp_header_path = "/tmp/clover/"; + + c.getHeaderSearchOpts().AddPath(tmp_header_path, + clang::frontend::Angled, + false, false); + + for (const auto &header : headers) + c.getPreprocessorOpts().addRemappedFile( + tmp_header_path + header.first, + ::llvm::MemoryBuffer::getMemBuffer(header.second).release()); + } + + // Tell clang to link this file before performing any + // optimizations. This is required so that we can replace calls + // to the OpenCL C barrier() builtin with calls to target + // intrinsics that have the noduplicate attribute. This + // attribute will prevent Clang from creating illegal uses of + // barrier() (e.g. Moving barrier() inside a conditional that is + // no executed by all threads) during its optimizaton passes. + if (use_libclc) + compat::add_link_bitcode_file(c.getCodeGenOpts(), + LIBCLC_LIBEXECDIR + dev.ir_target() + ".bc"); + + // Compile the code + clang::EmitLLVMOnlyAction act(&ctx); + if (!c.ExecuteAction(act)) + throw build_error(); + + return act.takeModule(); + } +} + +module +clover::llvm::compile_program(const std::string &source, + const header_map &headers, + const device &dev, + const std::string &opts, + std::string &r_log) { + if (has_flag(debug::clc)) + debug::log(".cl", "// Options: " + opts + '\n' + source); + + auto ctx = create_context(r_log); + auto c = create_compiler_instance(dev, dev.ir_target(), + tokenize(opts + " input.cl"), r_log); + auto mod = compile(*ctx, *c, "input.cl", source, headers, dev, opts, true, + r_log); + + if (has_flag(debug::llvm)) + debug::log(".ll", print_module_bitcode(*mod)); + + return build_module_library(*mod, module::section::text_intermediate); +} + +namespace { + void + optimize(Module &mod, unsigned optimization_level, + bool internalize_symbols) { + ::llvm::legacy::PassManager pm; + + // By default, the function internalizer pass will look for a function + // called "main" and then mark all other functions as internal. Marking + // functions as internal enables the optimizer to perform optimizations + // like function inlining and global dead-code elimination. + // + // When there is no "main" function in a module, the internalize pass will + // treat the module like a library, and it won't internalize any functions. + // Since there is no "main" function in our kernels, we need to tell + // the internalizer pass that this module is not a library by passing a + // list of kernel functions to the internalizer. The internalizer will + // treat the functions in the list as "main" functions and internalize + // all of the other functions. + if (internalize_symbols) { + std::vector<std::string> names = + map(std::mem_fn(&Function::getName), get_kernels(mod)); + pm.add(::llvm::createInternalizePass( + [=](const ::llvm::GlobalValue &gv) { + return std::find(names.begin(), names.end(), + gv.getName()) != names.end(); + })); + } + + ::llvm::PassManagerBuilder pmb; + pmb.OptLevel = optimization_level; + pmb.LibraryInfo = new ::llvm::TargetLibraryInfoImpl( + ::llvm::Triple(mod.getTargetTriple())); + pmb.populateModulePassManager(pm); + pm.run(mod); + } + + std::unique_ptr<Module> + link(LLVMContext &ctx, const clang::CompilerInstance &c, + const std::vector<module> &modules, std::string &r_log) { + std::unique_ptr<Module> mod { new Module("link", ctx) }; + std::unique_ptr< ::llvm::Linker> linker { new ::llvm::Linker(*mod) }; + + for (auto &m : modules) { + if (linker->linkInModule(parse_module_library(m, ctx, r_log))) + throw build_error(); + } + + return mod; + } +} + +module +clover::llvm::link_program(const std::vector<module> &modules, + const device &dev, const std::string &opts, + std::string &r_log) { + std::vector<std::string> options = tokenize(opts + " input.cl"); + const bool create_library = count("-create-library", options); + erase_if(equals("-create-library"), options); + + auto ctx = create_context(r_log); + auto c = create_compiler_instance(dev, dev.ir_target(), options, r_log); + auto mod = link(*ctx, *c, modules, r_log); + + optimize(*mod, c->getCodeGenOpts().OptimizationLevel, !create_library); + + static std::atomic_uint seq(0); + const std::string id = "." + mod->getModuleIdentifier() + "-" + + std::to_string(seq++); + + if (has_flag(debug::llvm)) + debug::log(id + ".ll", print_module_bitcode(*mod)); + + if (create_library) { + return build_module_library(*mod, module::section::text_library); + + } else if (dev.ir_format() == PIPE_SHADER_IR_NATIVE) { + if (has_flag(debug::native)) + debug::log(id + ".asm", print_module_native(*mod, dev.ir_target())); + + return build_module_native(*mod, dev.ir_target(), *c, r_log); + + } else { + unreachable("Unsupported IR."); + } +} + +#ifdef HAVE_CLOVER_SPIRV +module +clover::llvm::compile_to_spirv(const std::string &source, + const header_map &headers, + const device &dev, + const std::string &opts, + std::string &r_log) { + if (has_flag(debug::clc)) + debug::log(".cl", "// Options: " + opts + '\n' + source); + + auto ctx = create_context(r_log); + const std::string target = dev.address_bits() == 32u ? + "-spir-unknown-unknown" : + "-spir64-unknown-unknown"; + auto c = create_compiler_instance(dev, target, + tokenize(opts + " input.cl"), r_log); + auto mod = compile(*ctx, *c, "input.cl", source, headers, dev, opts, false, + r_log); + + if (has_flag(debug::llvm)) + debug::log(".ll", print_module_bitcode(*mod)); + + std::string error_msg; + if (!::llvm::regularizeLlvmForSpirv(mod.get(), error_msg)) { + r_log += "Failed to regularize LLVM IR for SPIR-V: " + error_msg + ".\n"; + throw error(CL_INVALID_VALUE); + } + + std::ostringstream os; + if (!::llvm::writeSpirv(mod.get(), os, error_msg)) { + r_log += "Translation from LLVM IR to SPIR-V failed: " + error_msg + ".\n"; + throw error(CL_INVALID_VALUE); + } + + const std::string osContent = os.str(); + std::vector<char> binary(osContent.begin(), osContent.end()); + if (binary.empty()) { + r_log += "Failed to retrieve SPIR-V binary.\n"; + throw error(CL_INVALID_VALUE); + } + + if (has_flag(debug::spirv)) + debug::log(".spvasm", spirv::print_module(binary, dev.device_version())); + + return spirv::compile_program(binary, dev, r_log); +} +#endif diff --git a/src/gallium/frontends/clover/llvm/invocation.hpp b/src/gallium/frontends/clover/llvm/invocation.hpp new file mode 100644 index 00000000000..1f0e9db2cf7 --- /dev/null +++ b/src/gallium/frontends/clover/llvm/invocation.hpp @@ -0,0 +1,54 @@ +// +// Copyright 2016 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_LLVM_INVOCATION_HPP +#define CLOVER_LLVM_INVOCATION_HPP + +#include "core/error.hpp" +#include "core/module.hpp" +#include "core/program.hpp" +#include "pipe/p_defines.h" + +namespace clover { + namespace llvm { + module compile_program(const std::string &source, + const header_map &headers, + const device &device, + const std::string &opts, + std::string &r_log); + + module link_program(const std::vector<module> &modules, + const device &device, + const std::string &opts, + std::string &r_log); + +#ifdef HAVE_CLOVER_SPIRV + module compile_to_spirv(const std::string &source, + const header_map &headers, + const device &dev, + const std::string &opts, + std::string &r_log); +#endif + } +} + +#endif diff --git a/src/gallium/frontends/clover/llvm/metadata.hpp b/src/gallium/frontends/clover/llvm/metadata.hpp new file mode 100644 index 00000000000..58042f4b4da --- /dev/null +++ b/src/gallium/frontends/clover/llvm/metadata.hpp @@ -0,0 +1,86 @@ +// +// Copyright 2016 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +/// +/// \file +/// Utility functions for LLVM IR metadata introspection. +/// + +#ifndef CLOVER_LLVM_METADATA_HPP +#define CLOVER_LLVM_METADATA_HPP + +#include "llvm/compat.hpp" +#include "util/algorithm.hpp" + +#include <vector> +#include <llvm/Config/llvm-config.h> +#include <llvm/IR/Module.h> +#include <llvm/IR/Metadata.h> + +namespace clover { + namespace llvm { + namespace detail { + inline bool + is_kernel(const ::llvm::Function &f) { + return f.getMetadata("kernel_arg_type"); + } + + inline iterator_range< ::llvm::MDNode::op_iterator> + get_kernel_metadata_operands(const ::llvm::Function &f, + const std::string &name) { + const auto data_node = f.getMetadata(name); + return range(data_node->op_begin(), data_node->op_end()); + } + } + + /// + /// Extract the string metadata node \p name corresponding to the kernel + /// argument given by \p arg. + /// + inline std::string + get_argument_metadata(const ::llvm::Function &f, + const ::llvm::Argument &arg, + const std::string &name) { + return ::llvm::cast< ::llvm::MDString>( + detail::get_kernel_metadata_operands(f, name)[arg.getArgNo()]) + ->getString().str(); + } + + /// + /// Return a vector with all CL kernel functions found in the LLVM + /// module \p mod. + /// + inline std::vector<const ::llvm::Function *> + get_kernels(const ::llvm::Module &mod) { + std::vector<const ::llvm::Function *> fs; + + for (auto &f : mod.getFunctionList()) { + if (detail::is_kernel(f)) + fs.push_back(&f); + } + + return fs; + } + } +} + +#endif diff --git a/src/gallium/frontends/clover/llvm/util.hpp b/src/gallium/frontends/clover/llvm/util.hpp new file mode 100644 index 00000000000..02e73e65071 --- /dev/null +++ b/src/gallium/frontends/clover/llvm/util.hpp @@ -0,0 +1,137 @@ +// +// Copyright 2012-2016 Francisco Jerez +// Copyright 2012-2016 Advanced Micro Devices, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_LLVM_UTIL_HPP +#define CLOVER_LLVM_UTIL_HPP + +#include "core/error.hpp" +#include "util/u_debug.h" + +#include <vector> +#include <fstream> +#include <iostream> +#include <sstream> + +namespace clover { + namespace llvm { + template<typename E> void + fail(std::string &r_log, E &&e, const std::string &s) { + r_log += s; + throw e; + } + + inline std::vector<std::string> + tokenize(const std::string &s) { + std::vector<std::string> ss; + std::ostringstream oss; + + // OpenCL programs can pass a quoted argument, most frequently the + // include path. This is useful so that path containing spaces is + // treated as a single argument instead of being split by the spaces. + // Additionally, the argument should also be unquoted before being + // passed to the compiler. We avoid using std::string::replace here to + // remove quotes, as the single and double quote characters can be a + // part of the file name. + bool escape_next = false; + bool in_quote_double = false; + bool in_quote_single = false; + + for (auto c : s) { + if (escape_next) { + oss.put(c); + escape_next = false; + } else if (c == '\\') { + escape_next = true; + } else if (c == '"' && !in_quote_single) { + in_quote_double = !in_quote_double; + } else if (c == '\'' && !in_quote_double) { + in_quote_single = !in_quote_single; + } else if (c != ' ' || in_quote_single || in_quote_double) { + oss.put(c); + } else if (oss.tellp() > 0) { + ss.emplace_back(oss.str()); + oss.str(""); + } + } + + if (oss.tellp() > 0) + ss.emplace_back(oss.str()); + + if (in_quote_double || in_quote_single) + throw invalid_build_options_error(); + + return ss; + } + + inline std::string + as_string(const std::vector<char> &v) { + return { v.begin(), v.end() }; + } + + struct target { + target(const std::string &s) : + cpu(s.begin(), s.begin() + s.find_first_of("-")), + triple(s.begin() + s.find_first_of("-") + 1, s.end()) {} + + std::string cpu; + std::string triple; + }; + + namespace debug { + enum flag { + clc = 1 << 0, + llvm = 1 << 1, + native = 1 << 2, + spirv = 1 << 3, + }; + + inline bool + has_flag(flag f) { + static const struct debug_named_value debug_options[] = { + { "clc", clc, "Dump the OpenCL C code for all kernels." }, + { "llvm", llvm, "Dump the generated LLVM IR for all kernels." }, + { "native", native, "Dump kernel assembly code for targets " + "specifying PIPE_SHADER_IR_NATIVE" }, + { "spirv", spirv, "Dump the generated SPIR-V for all kernels." }, + DEBUG_NAMED_VALUE_END + }; + static const unsigned flags = + debug_get_flags_option("CLOVER_DEBUG", debug_options, 0); + + return flags & f; + } + + inline void + log(const std::string &suffix, const std::string &s) { + const std::string path = debug_get_option("CLOVER_DEBUG_FILE", + "stderr"); + if (path == "stderr") + std::cerr << s; + else + std::ofstream(path + suffix, std::ios::app) << s; + } + } + } +} + +#endif diff --git a/src/gallium/frontends/clover/meson.build b/src/gallium/frontends/clover/meson.build new file mode 100644 index 00000000000..7606a6beaf6 --- /dev/null +++ b/src/gallium/frontends/clover/meson.build @@ -0,0 +1,164 @@ +# Copyright © 2017-2018 Intel Corporation + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +clover_cpp_args = [] +clover_opencl_cpp_args = [ + '-DCL_TARGET_OPENCL_VERSION=220', + '-DCL_USE_DEPRECATED_OPENCL_1_0_APIS', + '-DCL_USE_DEPRECATED_OPENCL_1_1_APIS', + '-DCL_USE_DEPRECATED_OPENCL_1_2_APIS', + '-DCL_USE_DEPRECATED_OPENCL_2_0_APIS', + '-DCL_USE_DEPRECATED_OPENCL_2_1_APIS' +] +clover_spirv_cpp_args = [] +clover_incs = [inc_include, inc_src, inc_gallium, inc_gallium_aux] + +# the CL header files declare attributes on the CL types. Compilers warn if +# we use them as template arguments. Disable the warning as there isn't +# anything we can do about it +if cpp.has_argument('-Wno-ignored-attributes') + clover_cpp_args += '-Wno-ignored-attributes' +endif + +if with_opencl_icd + clover_cpp_args += '-DHAVE_CLOVER_ICD' +endif + +if with_opencl_spirv + clover_spirv_cpp_args += '-DHAVE_CLOVER_SPIRV' +endif + +libclllvm = static_library( + 'clllvm', + files( + 'llvm/codegen/bitcode.cpp', + 'llvm/codegen/common.cpp', + 'llvm/codegen/native.cpp', + 'llvm/codegen.hpp', + 'llvm/compat.hpp', + 'llvm/invocation.cpp', + 'llvm/invocation.hpp', + 'llvm/metadata.hpp', + 'llvm/util.hpp', + ), + include_directories : clover_incs, + cpp_args : [ + clover_cpp_args, + clover_opencl_cpp_args, + clover_spirv_cpp_args, + cpp_vis_args, + '-DLIBCLC_INCLUDEDIR="@0@/"'.format(dep_clc.get_pkgconfig_variable('includedir')), + '-DLIBCLC_LIBEXECDIR="@0@/"'.format(dep_clc.get_pkgconfig_variable('libexecdir')), + '-DCLANG_RESOURCE_DIR="@0@"'.format(join_paths( + dep_llvm.get_configtool_variable('libdir'), 'clang', + dep_llvm.version(), 'include', + )), + ], + dependencies : [dep_llvm, dep_elf, dep_llvmspirvlib], + override_options : clover_cpp_std, +) + +libclspirv = static_library( + 'clspirv', + files('spirv/invocation.cpp', 'spirv/invocation.hpp'), + include_directories : clover_incs, + cpp_args : [clover_opencl_cpp_args, clover_spirv_cpp_args, cpp_vis_args], + dependencies : [dep_spirv_tools], + override_options : clover_cpp_std, +) + +libclnir = static_library( + 'clnir', + files('nir/invocation.cpp', 'nir/invocation.hpp'), + include_directories : [clover_incs, inc_mesa], + dependencies : idep_nir, + cpp_args : [clover_opencl_cpp_args, clover_spirv_cpp_args, cpp_vis_args], + override_options : clover_cpp_std, +) + +clover_files = files( + 'api/context.cpp', + 'api/device.cpp', + 'api/dispatch.cpp', + 'api/dispatch.hpp', + 'api/event.cpp', + 'api/interop.cpp', + 'api/kernel.cpp', + 'api/memory.cpp', + 'api/platform.cpp', + 'api/program.cpp', + 'api/queue.cpp', + 'api/sampler.cpp', + 'api/transfer.cpp', + 'api/util.hpp', + 'core/compiler.hpp', + 'core/context.cpp', + 'core/context.hpp', + 'core/device.cpp', + 'core/device.hpp', + 'core/error.hpp', + 'core/event.cpp', + 'core/event.hpp', + 'core/format.cpp', + 'core/format.hpp', + 'core/kernel.cpp', + 'core/kernel.hpp', + 'core/memory.cpp', + 'core/memory.hpp', + 'core/module.cpp', + 'core/module.hpp', + 'core/object.hpp', + 'core/platform.cpp', + 'core/platform.hpp', + 'core/program.cpp', + 'core/program.hpp', + 'core/property.hpp', + 'core/queue.cpp', + 'core/queue.hpp', + 'core/resource.cpp', + 'core/resource.hpp', + 'core/sampler.cpp', + 'core/sampler.hpp', + 'core/timestamp.cpp', + 'core/timestamp.hpp', + 'util/adaptor.hpp', + 'util/algebra.hpp', + 'util/algorithm.hpp', + 'util/factor.hpp', + 'util/functional.hpp', + 'util/lazy.hpp', + 'util/pointer.hpp', + 'util/range.hpp', + 'util/tuple.hpp', +) + +libclover = static_library( + 'clover', + [clover_files, sha1_h], + include_directories : clover_incs, + cpp_args : [ + clover_opencl_cpp_args, + clover_spirv_cpp_args, + clover_cpp_args, + cpp_vis_args + ], + link_with : [libclllvm, libclspirv, libclnir], + override_options : clover_cpp_std, +) diff --git a/src/gallium/frontends/clover/nir/invocation.cpp b/src/gallium/frontends/clover/nir/invocation.cpp new file mode 100644 index 00000000000..46440d96e09 --- /dev/null +++ b/src/gallium/frontends/clover/nir/invocation.cpp @@ -0,0 +1,173 @@ +// +// Copyright 2019 Karol Herbst +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "invocation.hpp" + +#include <tuple> + +#include "core/device.hpp" +#include "core/error.hpp" +#include "pipe/p_state.h" +#include "util/algorithm.hpp" +#include "util/functional.hpp" + +#include <compiler/glsl_types.h> +#include <compiler/nir/nir_serialize.h> +#include <compiler/spirv/nir_spirv.h> +#include <util/u_math.h> + +using namespace clover; + +#ifdef HAVE_CLOVER_SPIRV + +// Refs and unrefs the glsl_type_singleton. +static class glsl_type_ref { +public: + glsl_type_ref() { + glsl_type_singleton_init_or_ref(); + } + + ~glsl_type_ref() { + glsl_type_singleton_decref(); + } +} glsl_type_ref; + +static const nir_shader_compiler_options * +dev_get_nir_compiler_options(const device &dev) +{ + const void *co = dev.get_compiler_options(PIPE_SHADER_IR_NIR); + return static_cast<const nir_shader_compiler_options*>(co); +} + +module clover::nir::spirv_to_nir(const module &mod, const device &dev, + std::string &r_log) +{ + struct spirv_to_nir_options spirv_options = {}; + spirv_options.environment = NIR_SPIRV_OPENCL; + spirv_options.caps.address = true; + spirv_options.caps.float64 = true; + spirv_options.caps.int8 = true; + spirv_options.caps.int16 = true; + spirv_options.caps.int64 = true; + spirv_options.caps.kernel = true; + spirv_options.constant_as_global = true; + + module m; + // We only insert one section. + assert(mod.secs.size() == 1); + auto §ion = mod.secs[0]; + + module::resource_id section_id = 0; + for (const auto &sym : mod.syms) { + assert(sym.section == 0); + + const auto *binary = + reinterpret_cast<const pipe_binary_program_header *>(section.data.data()); + const uint32_t *data = reinterpret_cast<const uint32_t *>(binary->blob); + const size_t num_words = binary->num_bytes / 4; + const char *name = sym.name.c_str(); + auto *compiler_options = dev_get_nir_compiler_options(dev); + + nir_shader *nir = spirv_to_nir(data, num_words, nullptr, 0, + MESA_SHADER_KERNEL, name, + &spirv_options, compiler_options); + if (!nir) { + r_log += "Translation from SPIR-V to NIR for kernel \"" + sym.name + + "\" failed.\n"; + throw build_error(); + } + + nir->info.cs.local_size_variable = true; + nir_validate_shader(nir, "clover"); + + // Calculate input offsets. + unsigned offset = 0; + nir_foreach_variable_safe(var, &nir->inputs) { + offset = align(offset, glsl_get_cl_alignment(var->type)); + var->data.driver_location = offset; + offset += glsl_get_cl_size(var->type); + } + + // Inline all functions first. + // according to the comment on nir_inline_functions + NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp); + NIR_PASS_V(nir, nir_lower_returns); + NIR_PASS_V(nir, nir_inline_functions); + NIR_PASS_V(nir, nir_opt_deref); + + // Pick off the single entrypoint that we want. + foreach_list_typed_safe(nir_function, func, node, &nir->functions) { + if (!func->is_entrypoint) + exec_node_remove(&func->node); + } + assert(exec_list_length(&nir->functions) == 1); + + nir_validate_shader(nir, "clover after function inlining"); + + NIR_PASS_V(nir, nir_lower_variable_initializers, + static_cast<nir_variable_mode>(~nir_var_function_temp)); + + // copy propagate to prepare for lower_explicit_io + NIR_PASS_V(nir, nir_split_var_copies); + NIR_PASS_V(nir, nir_opt_copy_prop_vars); + NIR_PASS_V(nir, nir_lower_var_copies); + NIR_PASS_V(nir, nir_lower_vars_to_ssa); + NIR_PASS_V(nir, nir_opt_dce); + + nir_variable_mode modes = (nir_variable_mode)( + nir_var_shader_in | + nir_var_mem_global | + nir_var_mem_shared); + nir_address_format format = nir->info.cs.ptr_size == 64 ? + nir_address_format_64bit_global : nir_address_format_32bit_global; + NIR_PASS_V(nir, nir_lower_explicit_io, modes, format); + + NIR_PASS_V(nir, nir_lower_system_values); + if (compiler_options->lower_int64_options) + NIR_PASS_V(nir, nir_lower_int64, + compiler_options->lower_int64_options); + + NIR_PASS_V(nir, nir_opt_dce); + + struct blob blob; + blob_init(&blob); + nir_serialize(&blob, nir, false); + + const pipe_binary_program_header header { uint32_t(blob.size) }; + module::section text { section_id, module::section::text_executable, header.num_bytes, {} }; + text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header), + reinterpret_cast<const char *>(&header) + sizeof(header)); + text.data.insert(text.data.end(), blob.data, blob.data + blob.size); + + m.syms.emplace_back(sym.name, section_id, 0, sym.args); + m.secs.push_back(text); + section_id++; + } + return m; +} +#else +module clover::nir::spirv_to_nir(const module &mod, const device &dev, std::string &r_log) +{ + r_log += "SPIR-V support in clover is not enabled.\n"; + throw error(CL_LINKER_NOT_AVAILABLE); +} +#endif diff --git a/src/gallium/frontends/clover/nir/invocation.hpp b/src/gallium/frontends/clover/nir/invocation.hpp new file mode 100644 index 00000000000..41407a79765 --- /dev/null +++ b/src/gallium/frontends/clover/nir/invocation.hpp @@ -0,0 +1,36 @@ +// +// Copyright 2019 Karol Herbst +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_NIR_INVOCATION_HPP +#define CLOVER_NIR_INVOCATION_HPP + +#include "core/module.hpp" + +namespace clover { + class device; + namespace nir { + // converts a given spirv module to nir + module spirv_to_nir(const module &mod, const device &dev, std::string &r_log); + } +} + +#endif diff --git a/src/gallium/frontends/clover/spirv/invocation.cpp b/src/gallium/frontends/clover/spirv/invocation.cpp new file mode 100644 index 00000000000..01ced45c13b --- /dev/null +++ b/src/gallium/frontends/clover/spirv/invocation.cpp @@ -0,0 +1,740 @@ +// +// Copyright 2018 Pierre Moreau +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#include "invocation.hpp" + +#include <tuple> +#include <unordered_map> +#include <unordered_set> +#include <vector> + +#ifdef HAVE_CLOVER_SPIRV +#include <spirv-tools/libspirv.hpp> +#include <spirv-tools/linker.hpp> +#endif + +#include "core/error.hpp" +#include "core/platform.hpp" +#include "invocation.hpp" +#include "llvm/util.hpp" +#include "pipe/p_state.h" +#include "util/algorithm.hpp" +#include "util/functional.hpp" +#include "util/u_math.h" + +#include "compiler/spirv/spirv.h" + +#define SPIRV_HEADER_WORD_SIZE 5 + +using namespace clover; + +#ifdef HAVE_CLOVER_SPIRV +namespace { + + template<typename T> + T get(const char *source, size_t index) { + const uint32_t *word_ptr = reinterpret_cast<const uint32_t *>(source); + return static_cast<T>(word_ptr[index]); + } + + enum module::argument::type + convert_storage_class(SpvStorageClass storage_class, std::string &err) { + switch (storage_class) { + case SpvStorageClassFunction: + return module::argument::scalar; + case SpvStorageClassUniformConstant: + return module::argument::global; + case SpvStorageClassWorkgroup: + return module::argument::local; + case SpvStorageClassCrossWorkgroup: + return module::argument::global; + default: + err += "Invalid storage type " + std::to_string(storage_class) + "\n"; + throw build_error(); + } + } + + enum module::argument::type + convert_image_type(SpvId id, SpvDim dim, SpvAccessQualifier access, + std::string &err) { + if (dim == SpvDim2D && access == SpvAccessQualifierReadOnly) + return module::argument::image2d_rd; + else if (dim == SpvDim2D && access == SpvAccessQualifierWriteOnly) + return module::argument::image2d_wr; + else if (dim == SpvDim3D && access == SpvAccessQualifierReadOnly) + return module::argument::image3d_rd; + else if (dim == SpvDim3D && access == SpvAccessQualifierWriteOnly) + return module::argument::image3d_wr; + else { + err += "Unknown access qualifier " + std::to_string(access) + + " or dimension " + std::to_string(dim) + " for image " + + std::to_string(id) + ".\n"; + throw build_error(); + } + } + + module::section + make_text_section(const std::vector<char> &code, + enum module::section::type section_type) { + const pipe_binary_program_header header { uint32_t(code.size()) }; + module::section text { 0, section_type, header.num_bytes, {} }; + + text.data.insert(text.data.end(), reinterpret_cast<const char *>(&header), + reinterpret_cast<const char *>(&header) + sizeof(header)); + text.data.insert(text.data.end(), code.begin(), code.end()); + + return text; + } + + module + create_module_from_spirv(const std::vector<char> &source, + size_t pointer_byte_size, + std::string &err) { + const size_t length = source.size() / sizeof(uint32_t); + size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header + + std::string kernel_name; + size_t kernel_nb = 0u; + std::vector<module::argument> args; + + module m; + + std::unordered_map<SpvId, std::string> kernels; + std::unordered_map<SpvId, module::argument> types; + std::unordered_map<SpvId, SpvId> pointer_types; + std::unordered_map<SpvId, unsigned int> constants; + std::unordered_set<SpvId> packed_structures; + std::unordered_map<SpvId, std::vector<SpvFunctionParameterAttribute>> + func_param_attr_map; + + while (i < length) { + const auto inst = &source[i * sizeof(uint32_t)]; + const auto desc_word = get<uint32_t>(inst, 0); + const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); + const unsigned int num_operands = desc_word >> SpvWordCountShift; + + switch (opcode) { + case SpvOpEntryPoint: + if (get<SpvExecutionModel>(inst, 1) == SpvExecutionModelKernel) + kernels.emplace(get<SpvId>(inst, 2), + source.data() + (i + 3u) * sizeof(uint32_t)); + break; + + case SpvOpDecorate: { + const auto id = get<SpvId>(inst, 1); + const auto decoration = get<SpvDecoration>(inst, 2); + if (decoration == SpvDecorationCPacked) + packed_structures.emplace(id); + else if (decoration == SpvDecorationFuncParamAttr) { + const auto attribute = + get<SpvFunctionParameterAttribute>(inst, 3u); + func_param_attr_map[id].push_back(attribute); + } + break; + } + + case SpvOpGroupDecorate: { + const auto group_id = get<SpvId>(inst, 1); + if (packed_structures.count(group_id)) { + for (unsigned int i = 2u; i < num_operands; ++i) + packed_structures.emplace(get<SpvId>(inst, i)); + } + const auto func_param_attr_iter = + func_param_attr_map.find(group_id); + if (func_param_attr_iter != func_param_attr_map.end()) { + for (unsigned int i = 2u; i < num_operands; ++i) + func_param_attr_map.emplace(get<SpvId>(inst, i), + func_param_attr_iter->second); + } + break; + } + + case SpvOpConstant: + // We only care about constants that represent the size of arrays. + // If they are passed as argument, they will never be more than + // 4GB-wide, and even if they did, a clover::module::argument size + // is represented by an int. + constants[get<SpvId>(inst, 2)] = get<unsigned int>(inst, 3u); + break; + + case SpvOpTypeInt: // FALLTHROUGH + case SpvOpTypeFloat: { + const auto size = get<uint32_t>(inst, 2) / 8u; + types[get<SpvId>(inst, 1)] = { module::argument::scalar, size, + size, size, + module::argument::zero_ext }; + break; + } + + case SpvOpTypeArray: { + const auto id = get<SpvId>(inst, 1); + const auto type_id = get<SpvId>(inst, 2); + const auto types_iter = types.find(type_id); + if (types_iter == types.end()) + break; + + const auto constant_id = get<SpvId>(inst, 3); + const auto constants_iter = constants.find(constant_id); + if (constants_iter == constants.end()) { + err += "Constant " + std::to_string(constant_id) + + " is missing\n"; + throw build_error(); + } + const auto elem_size = types_iter->second.size; + const auto elem_nbs = constants_iter->second; + const auto size = elem_size * elem_nbs; + types[id] = { module::argument::scalar, size, size, + types_iter->second.target_align, + module::argument::zero_ext }; + break; + } + + case SpvOpTypeStruct: { + const auto id = get<SpvId>(inst, 1); + const bool is_packed = packed_structures.count(id); + + unsigned struct_size = 0u; + unsigned struct_align = 1u; + for (unsigned j = 2u; j < num_operands; ++j) { + const auto type_id = get<SpvId>(inst, j); + const auto types_iter = types.find(type_id); + + // If a type was not found, that means it is not one of the + // types allowed as kernel arguments. And since the module has + // been validated, this means this type is not used for kernel + // arguments, and therefore can be ignored. + if (types_iter == types.end()) + break; + + const auto alignment = is_packed ? 1u + : types_iter->second.target_align; + const auto padding = (-struct_size) & (alignment - 1u); + struct_size += padding + types_iter->second.target_size; + struct_align = std::max(struct_align, alignment); + } + struct_size += (-struct_size) & (struct_align - 1u); + types[id] = { module::argument::scalar, struct_size, struct_size, + struct_align, module::argument::zero_ext }; + break; + } + + case SpvOpTypeVector: { + const auto id = get<SpvId>(inst, 1); + const auto type_id = get<SpvId>(inst, 2); + const auto types_iter = types.find(type_id); + + // If a type was not found, that means it is not one of the + // types allowed as kernel arguments. And since the module has + // been validated, this means this type is not used for kernel + // arguments, and therefore can be ignored. + if (types_iter == types.end()) + break; + + const auto elem_size = types_iter->second.size; + const auto elem_nbs = get<uint32_t>(inst, 3); + const auto size = elem_size * elem_nbs; + types[id] = { module::argument::scalar, size, size, size, + module::argument::zero_ext }; + break; + } + + case SpvOpTypeForwardPointer: // FALLTHROUGH + case SpvOpTypePointer: { + const auto id = get<SpvId>(inst, 1); + const auto storage_class = get<SpvStorageClass>(inst, 2); + // Input means this is for a builtin variable, which can not be + // passed as an argument to a kernel. + if (storage_class == SpvStorageClassInput) + break; + types[id] = { convert_storage_class(storage_class, err), + sizeof(cl_mem), + static_cast<module::size_t>(pointer_byte_size), + static_cast<module::size_t>(pointer_byte_size), + module::argument::zero_ext }; + if (opcode == SpvOpTypePointer) + pointer_types[id] = get<SpvId>(inst, 3); + break; + } + + case SpvOpTypeSampler: + types[get<SpvId>(inst, 1)] = { module::argument::sampler, + sizeof(cl_sampler) }; + break; + + case SpvOpTypeImage: { + const auto id = get<SpvId>(inst, 1); + const auto dim = get<SpvDim>(inst, 3); + const auto access = get<SpvAccessQualifier>(inst, 9); + types[id] = { convert_image_type(id, dim, access, err), + sizeof(cl_mem), sizeof(cl_mem), sizeof(cl_mem), + module::argument::zero_ext }; + break; + } + + case SpvOpTypePipe: // FALLTHROUGH + case SpvOpTypeQueue: { + err += "TypePipe and TypeQueue are valid SPIR-V 1.0 types, but are " + "not available in the currently supported OpenCL C version." + "\n"; + throw build_error(); + } + + case SpvOpFunction: { + const auto kernels_iter = kernels.find(get<SpvId>(inst, 2)); + if (kernels_iter != kernels.end()) + kernel_name = kernels_iter->second; + break; + } + + case SpvOpFunctionParameter: { + if (kernel_name.empty()) + break; + + const auto type_id = get<SpvId>(inst, 1); + auto arg = types.find(type_id)->second; + const auto &func_param_attr_iter = + func_param_attr_map.find(get<SpvId>(inst, 2)); + if (func_param_attr_iter != func_param_attr_map.end()) { + for (auto &i : func_param_attr_iter->second) { + switch (i) { + case SpvFunctionParameterAttributeSext: + arg.ext_type = module::argument::sign_ext; + break; + case SpvFunctionParameterAttributeZext: + arg.ext_type = module::argument::zero_ext; + break; + case SpvFunctionParameterAttributeByVal: { + const SpvId ptr_type_id = + pointer_types.find(type_id)->second; + arg = types.find(ptr_type_id)->second; + break; + } + default: + break; + } + } + } + args.emplace_back(arg); + break; + } + + case SpvOpFunctionEnd: + if (kernel_name.empty()) + break; + m.syms.emplace_back(kernel_name, 0, kernel_nb, args); + ++kernel_nb; + kernel_name.clear(); + args.clear(); + break; + + default: + break; + } + + i += num_operands; + } + + m.secs.push_back(make_text_section(source, + module::section::text_intermediate)); + return m; + } + + bool + check_capabilities(const device &dev, const std::vector<char> &source, + std::string &r_log) { + const size_t length = source.size() / sizeof(uint32_t); + size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header + + while (i < length) { + const auto desc_word = get<uint32_t>(source.data(), i); + const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); + const unsigned int num_operands = desc_word >> SpvWordCountShift; + + if (opcode != SpvOpCapability) + break; + + const auto capability = get<SpvCapability>(source.data(), i + 1u); + switch (capability) { + // Mandatory capabilities + case SpvCapabilityAddresses: + case SpvCapabilityFloat16Buffer: + case SpvCapabilityGroups: + case SpvCapabilityInt64: + case SpvCapabilityInt16: + case SpvCapabilityInt8: + case SpvCapabilityKernel: + case SpvCapabilityLinkage: + case SpvCapabilityVector16: + break; + // Optional capabilities + case SpvCapabilityImageBasic: + case SpvCapabilityLiteralSampler: + case SpvCapabilitySampled1D: + case SpvCapabilityImage1D: + case SpvCapabilitySampledBuffer: + case SpvCapabilityImageBuffer: + if (!dev.image_support()) { + r_log += "Capability 'ImageBasic' is not supported.\n"; + return false; + } + break; + case SpvCapabilityFloat64: + if (!dev.has_doubles()) { + r_log += "Capability 'Float64' is not supported.\n"; + return false; + } + break; + // Enabled through extensions + case SpvCapabilityFloat16: + if (!dev.has_halves()) { + r_log += "Capability 'Float16' is not supported.\n"; + return false; + } + break; + case SpvCapabilityInt64Atomics: + if (!dev.has_int64_atomics()) { + r_log += "Capability 'Int64Atomics' is not supported.\n"; + return false; + } + break; + default: + r_log += "Capability '" + std::to_string(capability) + + "' is not supported.\n"; + return false; + } + + i += num_operands; + } + + return true; + } + + bool + check_extensions(const device &dev, const std::vector<char> &source, + std::string &r_log) { + const size_t length = source.size() / sizeof(uint32_t); + size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header + + while (i < length) { + const auto desc_word = get<uint32_t>(source.data(), i); + const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); + const unsigned int num_operands = desc_word >> SpvWordCountShift; + + if (opcode == SpvOpCapability) { + i += num_operands; + continue; + } + if (opcode != SpvOpExtension) + break; + + const char *extension = source.data() + (i + 1u) * sizeof(uint32_t); + const std::string device_extensions = dev.supported_extensions(); + const std::string platform_extensions = + dev.platform.supported_extensions(); + if (device_extensions.find(extension) == std::string::npos && + platform_extensions.find(extension) == std::string::npos) { + r_log += "Extension '" + std::string(extension) + + "' is not supported.\n"; + return false; + } + + i += num_operands; + } + + return true; + } + + bool + check_memory_model(const device &dev, const std::vector<char> &source, + std::string &r_log) { + const size_t length = source.size() / sizeof(uint32_t); + size_t i = SPIRV_HEADER_WORD_SIZE; // Skip header + + while (i < length) { + const auto desc_word = get<uint32_t>(source.data(), i); + const auto opcode = static_cast<SpvOp>(desc_word & SpvOpCodeMask); + const unsigned int num_operands = desc_word >> SpvWordCountShift; + + switch (opcode) { + case SpvOpMemoryModel: + switch (get<SpvAddressingModel>(source.data(), i + 1u)) { + case SpvAddressingModelPhysical32: + return dev.address_bits() == 32; + case SpvAddressingModelPhysical64: + return dev.address_bits() == 64; + default: + unreachable("Only Physical32 and Physical64 are valid for OpenCL, and the binary was already validated"); + return false; + } + break; + default: + break; + } + + i += num_operands; + } + + return false; + } + + // Copies the input binary and convert it to the endianness of the host CPU. + std::vector<char> + spirv_to_cpu(const std::vector<char> &binary) + { + const uint32_t first_word = get<uint32_t>(binary.data(), 0u); + if (first_word == SpvMagicNumber) + return binary; + + std::vector<char> cpu_endianness_binary(binary.size()); + for (size_t i = 0; i < (binary.size() / 4u); ++i) { + const uint32_t word = get<uint32_t>(binary.data(), i); + reinterpret_cast<uint32_t *>(cpu_endianness_binary.data())[i] = + util_bswap32(word); + } + + return cpu_endianness_binary; + } + +#ifdef HAVE_CLOVER_SPIRV + std::string + format_validator_msg(spv_message_level_t level, const char * /* source */, + const spv_position_t &position, const char *message) { + std::string level_str; + switch (level) { + case SPV_MSG_FATAL: + level_str = "Fatal"; + break; + case SPV_MSG_INTERNAL_ERROR: + level_str = "Internal error"; + break; + case SPV_MSG_ERROR: + level_str = "Error"; + break; + case SPV_MSG_WARNING: + level_str = "Warning"; + break; + case SPV_MSG_INFO: + level_str = "Info"; + break; + case SPV_MSG_DEBUG: + level_str = "Debug"; + break; + } + return "[" + level_str + "] At word No." + + std::to_string(position.index) + ": \"" + message + "\"\n"; + } + + spv_target_env + convert_opencl_str_to_target_env(const std::string &opencl_version) { + if (opencl_version == "2.2") { + return SPV_ENV_OPENCL_2_2; + } else if (opencl_version == "2.1") { + return SPV_ENV_OPENCL_2_1; + } else if (opencl_version == "2.0") { + return SPV_ENV_OPENCL_2_0; + } else if (opencl_version == "1.2" || + opencl_version == "1.1" || + opencl_version == "1.0") { + // SPIR-V is only defined for OpenCL >= 1.2, however some drivers + // might use it with OpenCL 1.0 and 1.1. + return SPV_ENV_OPENCL_1_2; + } else { + throw build_error("Invalid OpenCL version"); + } + } +#endif + +} + +module +clover::spirv::compile_program(const std::vector<char> &binary, + const device &dev, std::string &r_log) { + std::vector<char> source = spirv_to_cpu(binary); + + if (!is_valid_spirv(source, dev.device_version(), r_log)) + throw build_error(); + + if (!check_capabilities(dev, source, r_log)) + throw build_error(); + if (!check_extensions(dev, source, r_log)) + throw build_error(); + if (!check_memory_model(dev, source, r_log)) + throw build_error(); + + return create_module_from_spirv(source, + dev.address_bits() == 32 ? 4u : 8u, r_log); +} + +module +clover::spirv::link_program(const std::vector<module> &modules, + const device &dev, const std::string &opts, + std::string &r_log) { + std::vector<std::string> options = clover::llvm::tokenize(opts); + + bool create_library = false; + + std::string ignored_options; + for (const std::string &option : options) { + if (option == "-create-library") { + create_library = true; + } else { + ignored_options += "'" + option + "' "; + } + } + if (!ignored_options.empty()) { + r_log += "Ignoring the following link options: " + ignored_options + + "\n"; + } + + spvtools::LinkerOptions linker_options; + linker_options.SetCreateLibrary(create_library); + + module m; + + const auto section_type = create_library ? module::section::text_library : + module::section::text_executable; + + std::vector<const uint32_t *> sections; + sections.reserve(modules.size()); + std::vector<size_t> lengths; + lengths.reserve(modules.size()); + + auto const validator_consumer = [&r_log](spv_message_level_t level, + const char *source, + const spv_position_t &position, + const char *message) { + r_log += format_validator_msg(level, source, position, message); + }; + + for (const auto &mod : modules) { + const auto &msec = find([](const module::section &sec) { + return sec.type == module::section::text_intermediate || + sec.type == module::section::text_library; + }, mod.secs); + + const auto c_il = ((struct pipe_binary_program_header*)msec.data.data())->blob; + const auto length = msec.size; + + sections.push_back(reinterpret_cast<const uint32_t *>(c_il)); + lengths.push_back(length / sizeof(uint32_t)); + } + + std::vector<uint32_t> linked_binary; + + const std::string opencl_version = dev.device_version(); + const spv_target_env target_env = + convert_opencl_str_to_target_env(opencl_version); + + const spvtools::MessageConsumer consumer = validator_consumer; + spvtools::Context context(target_env); + context.SetMessageConsumer(std::move(consumer)); + + if (Link(context, sections.data(), lengths.data(), sections.size(), + &linked_binary, linker_options) != SPV_SUCCESS) + throw error(CL_LINK_PROGRAM_FAILURE); + + std::vector<char> final_binary{ + reinterpret_cast<char *>(linked_binary.data()), + reinterpret_cast<char *>(linked_binary.data() + + linked_binary.size()) }; + if (!is_valid_spirv(final_binary, opencl_version, r_log)) + throw error(CL_LINK_PROGRAM_FAILURE); + + for (const auto &mod : modules) + m.syms.insert(m.syms.end(), mod.syms.begin(), mod.syms.end()); + + m.secs.emplace_back(make_text_section(final_binary, section_type)); + + return m; +} + +bool +clover::spirv::is_valid_spirv(const std::vector<char> &binary, + const std::string &opencl_version, + std::string &r_log) { + auto const validator_consumer = + [&r_log](spv_message_level_t level, const char *source, + const spv_position_t &position, const char *message) { + r_log += format_validator_msg(level, source, position, message); + }; + + const spv_target_env target_env = + convert_opencl_str_to_target_env(opencl_version); + spvtools::SpirvTools spvTool(target_env); + spvTool.SetMessageConsumer(validator_consumer); + + return spvTool.Validate(reinterpret_cast<const uint32_t *>(binary.data()), + binary.size() / 4u); +} + +std::string +clover::spirv::print_module(const std::vector<char> &binary, + const std::string &opencl_version) { + const spv_target_env target_env = + convert_opencl_str_to_target_env(opencl_version); + spvtools::SpirvTools spvTool(target_env); + spv_context spvContext = spvContextCreate(target_env); + if (!spvContext) + return "Failed to create an spv_context for disassembling the module."; + + spv_text disassembly; + spvBinaryToText(spvContext, + reinterpret_cast<const uint32_t *>(binary.data()), + binary.size() / 4u, SPV_BINARY_TO_TEXT_OPTION_NONE, + &disassembly, nullptr); + spvContextDestroy(spvContext); + + const std::string disassemblyStr = disassembly->str; + spvTextDestroy(disassembly); + + return disassemblyStr; +} + +#else +bool +clover::spirv::is_valid_spirv(const std::vector<char> &/*binary*/, + const std::string &/*opencl_version*/, + std::string &/*r_log*/) { + return false; +} + +module +clover::spirv::compile_program(const std::vector<char> &binary, + const device &dev, std::string &r_log) { + r_log += "SPIR-V support in clover is not enabled.\n"; + throw build_error(); +} + +module +clover::spirv::link_program(const std::vector<module> &/*modules*/, + const device &/*dev*/, const std::string &/*opts*/, + std::string &r_log) { + r_log += "SPIR-V support in clover is not enabled.\n"; + throw error(CL_LINKER_NOT_AVAILABLE); +} + +std::string +clover::spirv::print_module(const std::vector<char> &binary, + const std::string &opencl_version) { + return std::string(); +} +#endif diff --git a/src/gallium/frontends/clover/spirv/invocation.hpp b/src/gallium/frontends/clover/spirv/invocation.hpp new file mode 100644 index 00000000000..472d8c0de71 --- /dev/null +++ b/src/gallium/frontends/clover/spirv/invocation.hpp @@ -0,0 +1,56 @@ +// +// Copyright 2018 Pierre Moreau +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_SPIRV_INVOCATION_HPP +#define CLOVER_SPIRV_INVOCATION_HPP + +#include "core/context.hpp" +#include "core/module.hpp" +#include "core/program.hpp" + +namespace clover { + namespace spirv { + // Returns whether the given binary is considered valid for the given + // OpenCL version. + // + // It uses SPIRV-Tools validator to do the validation, and potential + // warnings and errors are appended to |r_log|. + bool is_valid_spirv(const std::vector<char> &binary, + const std::string &opencl_version, + std::string &r_log); + + // Creates a clover module out of the given SPIR-V binary. + module compile_program(const std::vector<char> &binary, + const device &dev, std::string &r_log); + + // Combines multiple clover modules into a single one, resolving + // link dependencies between them. + module link_program(const std::vector<module> &modules, const device &dev, + const std::string &opts, std::string &r_log); + + // Returns a textual representation of the given binary. + std::string print_module(const std::vector<char> &binary, + const std::string &opencl_version); + } +} + +#endif diff --git a/src/gallium/frontends/clover/util/adaptor.hpp b/src/gallium/frontends/clover/util/adaptor.hpp new file mode 100644 index 00000000000..e9035968573 --- /dev/null +++ b/src/gallium/frontends/clover/util/adaptor.hpp @@ -0,0 +1,183 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_UTIL_ADAPTOR_HPP +#define CLOVER_UTIL_ADAPTOR_HPP + +#include <iterator> + +#include "util/tuple.hpp" +#include "util/pointer.hpp" +#include "util/functional.hpp" + +namespace clover { + namespace detail { + /// + /// Implementation of the iterator concept that transforms the + /// value of the source iterators \a Is on dereference by use of + /// a functor \a F. + /// + /// The exact category of the resulting iterator should be the + /// least common denominator of the source iterator categories. + /// + template<typename F, typename... Is> + class iterator_adaptor { + public: + typedef std::forward_iterator_tag iterator_category; + typedef typename std::result_of< + F(typename std::iterator_traits<Is>::reference...) + >::type reference; + typedef typename std::remove_reference<reference>::type value_type; + typedef pseudo_ptr<value_type> pointer; + typedef std::ptrdiff_t difference_type; + + iterator_adaptor() { + } + + iterator_adaptor(F f, std::tuple<Is...> &&its) : + f(f), its(std::move(its)) { + } + + reference + operator*() const { + return tuple::apply(f, tuple::map(derefs(), its)); + } + + iterator_adaptor & + operator++() { + tuple::map(preincs(), its); + return *this; + } + + iterator_adaptor + operator++(int) { + auto jt = *this; + ++*this; + return jt; + } + + bool + operator==(const iterator_adaptor &jt) const { + return its == jt.its; + } + + bool + operator!=(const iterator_adaptor &jt) const { + return its != jt.its; + } + + pointer + operator->() const { + return { **this }; + } + + iterator_adaptor & + operator--() { + tuple::map(predecs(), its); + return *this; + } + + iterator_adaptor + operator--(int) { + auto jt = *this; + --*this; + return jt; + } + + iterator_adaptor & + operator+=(difference_type n) { + tuple::map(advances_by(n), its); + return *this; + } + + iterator_adaptor & + operator-=(difference_type n) { + tuple::map(advances_by(-n), its); + return *this; + } + + iterator_adaptor + operator+(difference_type n) const { + auto jt = *this; + jt += n; + return jt; + } + + iterator_adaptor + operator-(difference_type n) const { + auto jt = *this; + jt -= n; + return jt; + } + + difference_type + operator-(const iterator_adaptor &jt) const { + return std::get<0>(its) - std::get<0>(jt.its); + } + + reference + operator[](difference_type n) const { + return *(*this + n); + } + + bool + operator<(iterator_adaptor &jt) const { + return *this - jt < 0; + } + + bool + operator>(iterator_adaptor &jt) const { + return *this - jt > 0; + } + + bool + operator>=(iterator_adaptor &jt) const { + return !(*this < jt); + } + + bool + operator<=(iterator_adaptor &jt) const { + return !(*this > jt); + } + + protected: + F f; + std::tuple<Is...> its; + }; + + template<typename F, typename... Is> + iterator_adaptor<F, Is...> + operator+(typename iterator_adaptor<F, Is...>::difference_type n, + const iterator_adaptor<F, Is...> &jt) { + return (jt + n); + } + + template<typename F, typename... Is> + iterator_adaptor<F, Is...> + operator-(typename iterator_adaptor<F, Is...>::difference_type n, + const iterator_adaptor<F, Is...> &jt) { + return (jt - n); + } + } +} + +#endif diff --git a/src/gallium/frontends/clover/util/algebra.hpp b/src/gallium/frontends/clover/util/algebra.hpp new file mode 100644 index 00000000000..43a9d8bbf5f --- /dev/null +++ b/src/gallium/frontends/clover/util/algebra.hpp @@ -0,0 +1,160 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_UTIL_ALGEBRA_HPP +#define CLOVER_UTIL_ALGEBRA_HPP + +#include <type_traits> + +#include "util/range.hpp" +#include "util/functional.hpp" + +namespace clover { + /// + /// Class that identifies vectors (in the linear-algebraic sense). + /// + /// There should be a definition of this class for each type that + /// makes sense as vector arithmetic operand. + /// + template<typename V, typename = void> + struct vector_traits; + + /// + /// References of vectors are vectors. + /// + template<typename T> + struct vector_traits<T &, typename vector_traits<T>::enable> { + typedef void enable; + }; + + /// + /// Constant vectors are vectors. + /// + template<typename T> + struct vector_traits<const T, typename vector_traits<T>::enable> { + typedef void enable; + }; + + /// + /// Arrays of arithmetic types are vectors. + /// + template<typename T, std::size_t N> + struct vector_traits<std::array<T, N>, + typename std::enable_if< + std::is_arithmetic<T>::value>::type> { + typedef void enable; + }; + + namespace detail { + template<typename... Ts> + struct are_defined { + typedef void enable; + }; + } + + /// + /// The result of mapping a vector is a vector. + /// + template<typename F, typename... Vs> + struct vector_traits<adaptor_range<F, Vs...>, + typename detail::are_defined< + typename vector_traits<Vs>::enable...>::enable> { + typedef void enable; + }; + + /// + /// Vector sum. + /// + template<typename U, typename V, + typename = typename vector_traits<U>::enable, + typename = typename vector_traits<V>::enable> + adaptor_range<plus, U, V> + operator+(U &&u, V &&v) { + return map(plus(), std::forward<U>(u), std::forward<V>(v)); + } + + /// + /// Vector difference. + /// + template<typename U, typename V, + typename = typename vector_traits<U>::enable, + typename = typename vector_traits<V>::enable> + adaptor_range<minus, U, V> + operator-(U &&u, V &&v) { + return map(minus(), std::forward<U>(u), std::forward<V>(v)); + } + + /// + /// Scalar multiplication. + /// + template<typename U, typename T, + typename = typename vector_traits<U>::enable> + adaptor_range<multiplies_by_t<T>, U> + operator*(U &&u, T &&a) { + return map(multiplies_by<T>(std::forward<T>(a)), std::forward<U>(u)); + } + + /// + /// Scalar multiplication. + /// + template<typename U, typename T, + typename = typename vector_traits<U>::enable> + adaptor_range<multiplies_by_t<T>, U> + operator*(T &&a, U &&u) { + return map(multiplies_by<T>(std::forward<T>(a)), std::forward<U>(u)); + } + + /// + /// Additive inverse. + /// + template<typename U, + typename = typename vector_traits<U>::enable> + adaptor_range<negate, U> + operator-(U &&u) { + return map(negate(), std::forward<U>(u)); + } + + namespace detail { + template<typename U, typename V> + using dot_type = typename std::common_type< + typename std::remove_reference<U>::type::value_type, + typename std::remove_reference<V>::type::value_type + >::type; + } + + /// + /// Dot product of two vectors. + /// + /// It can also do matrix multiplication if \a u or \a v is a + /// vector of vectors. + /// + template<typename U, typename V, + typename = typename vector_traits<U>::enable, + typename = typename vector_traits<V>::enable> + detail::dot_type<U, V> + dot(U &&u, V &&v) { + return fold(plus(), detail::dot_type<U, V>(), + map(multiplies(), u, v)); + } +} + +#endif diff --git a/src/gallium/frontends/clover/util/algorithm.hpp b/src/gallium/frontends/clover/util/algorithm.hpp new file mode 100644 index 00000000000..1658458ee18 --- /dev/null +++ b/src/gallium/frontends/clover/util/algorithm.hpp @@ -0,0 +1,218 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_UTIL_ALGORITHM_HPP +#define CLOVER_UTIL_ALGORITHM_HPP + +#include <algorithm> +#include <stdexcept> + +#include "util/range.hpp" +#include "util/functional.hpp" + +namespace clover { + namespace detail { + template<typename R> + using preferred_reference_type = decltype(*std::declval<R>().begin()); + } + + /// + /// Return the first element in a range. + /// + template<typename R> + detail::preferred_reference_type<R> + head(R &&r) { + assert(!r.empty()); + return r.front(); + } + + /// + /// Return all elements in a range but the first. + /// + template<typename R> + slice_range<R> + tail(R &&r) { + assert(!r.empty()); + return { std::forward<R>(r), 1, r.size() }; + } + + /// + /// Return the only element in a range. + /// + template<typename R> + detail::preferred_reference_type<R> + unique(R &&r) { + if (r.size() != 1) + throw std::out_of_range(""); + + return r.front(); + } + + /// + /// Combine a variable number of ranges element-wise in a single + /// range of tuples. + /// + template<typename... Rs> + adaptor_range<zips, Rs...> + zip(Rs &&... rs) { + return map(zips(), std::forward<Rs>(rs)...); + } + + /// + /// Evaluate the elements of a range. + /// + /// Useful because most of the range algorithms evaluate their + /// result lazily. + /// + template<typename R> + void + eval(R &&r) { + for (auto i = r.begin(), e = r.end(); i != e; ++i) + *i; + } + + /// + /// Apply functor \a f element-wise on a variable number of ranges + /// \a rs. + /// + /// The functor \a f should take as many arguments as ranges are + /// provided. + /// + template<typename F, typename... Rs> + void + for_each(F &&f, Rs &&... rs) { + eval(map(std::forward<F>(f), std::forward<Rs>(rs)...)); + } + + /// + /// Copy all elements from range \a r into an output container + /// starting from iterator \a i. + /// + template<typename R, typename I> + void + copy(R &&r, I i) { + for (detail::preferred_reference_type<R> x : r) + *(i++) = x; + } + + /// + /// Reduce the elements of range \a r by applying functor \a f + /// element by element. + /// + /// \a f should take an accumulator value (which is initialized to + /// \a a) and an element value as arguments, and return an updated + /// accumulator value. + /// + /// \returns The final value of the accumulator. + /// + template<typename F, typename A, typename R> + A + fold(F &&f, A a, R &&r) { + for (detail::preferred_reference_type<R> x : r) + a = f(a, x); + + return a; + } + + /// + /// Return how many elements of range \a r are equal to \a x. + /// + template<typename T, typename R> + typename std::remove_reference<R>::type::size_type + count(T &&x, R &&r) { + typename std::remove_reference<R>::type::size_type n = 0; + + for (detail::preferred_reference_type<R> y : r) { + if (x == y) + n++; + } + + return n; + } + + /// + /// Return the first element in range \a r for which predicate \a f + /// evaluates to true. + /// + template<typename F, typename R> + detail::preferred_reference_type<R> + find(F &&f, R &&r) { + for (detail::preferred_reference_type<R> x : r) { + if (f(x)) + return x; + } + + throw std::out_of_range(""); + } + + /// + /// Return true if the element-wise application of predicate \a f + /// on \a rs evaluates to true for all elements. + /// + template<typename F, typename... Rs> + bool + all_of(F &&f, Rs &&... rs) { + for (auto b : map(f, rs...)) { + if (!b) + return false; + } + + return true; + } + + /// + /// Return true if the element-wise application of predicate \a f + /// on \a rs evaluates to true for any element. + /// + template<typename F, typename... Rs> + bool + any_of(F &&f, Rs &&... rs) { + for (auto b : map(f, rs...)) { + if (b) + return true; + } + + return false; + } + + /// + /// Erase elements for which predicate \a f evaluates to true from + /// container \a r. + /// + template<typename F, typename R> + void + erase_if(F &&f, R &&r) { + auto i = r.begin(), e = r.end(); + + for (auto j = r.begin(); j != e; ++j) { + if (!f(*j)) { + if (j != i) + *i = std::move(*j); + ++i; + } + } + + r.erase(i, e); + } +} + +#endif diff --git a/src/gallium/frontends/clover/util/factor.hpp b/src/gallium/frontends/clover/util/factor.hpp new file mode 100644 index 00000000000..76d3bfe343f --- /dev/null +++ b/src/gallium/frontends/clover/util/factor.hpp @@ -0,0 +1,131 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_UTIL_FACTOR_HPP +#define CLOVER_UTIL_FACTOR_HPP + +#include "util/range.hpp" + +namespace clover { + namespace factor { + /// + /// Calculate all prime integer factors of \p x. + /// + /// If \p limit is non-zero, terminate early as soon as enough + /// factors have been collected to reach the product \p limit. + /// + template<typename T> + std::vector<T> + find_integer_prime_factors(T x, T limit = 0) + { + const T max_d = (limit > 0 && limit < x ? limit : x); + const T min_x = x / max_d; + std::vector<T> factors; + + for (T d = 2; d <= max_d && x > min_x; d++) { + if (x % d == 0) { + for (; x % d == 0; x /= d); + factors.push_back(d); + } + } + + return factors; + } + + namespace detail { + /// + /// Walk the power set of prime factors of the n-dimensional + /// integer array \p grid subject to the constraints given by + /// \p limits. + /// + template<typename T> + std::pair<T, std::vector<T>> + next_grid_factor(const std::pair<T, std::vector<T>> &limits, + const std::vector<T> &grid, + const std::vector<std::vector<T>> &factors, + std::pair<T, std::vector<T>> block, + unsigned d = 0, unsigned i = 0) { + if (d >= factors.size()) { + // We're done. + return {}; + + } else if (i >= factors[d].size()) { + // We're done with this grid dimension, try the next. + return next_grid_factor(limits, grid, factors, + std::move(block), d + 1, 0); + + } else { + T f = factors[d][i]; + + // Try the next power of this factor. + block.first *= f; + block.second[d] *= f; + + if (block.first <= limits.first && + block.second[d] <= limits.second[d] && + grid[d] % block.second[d] == 0) { + // We've found a valid grid divisor. + return block; + + } else { + // Overflow, back off to the zeroth power, + while (block.second[d] % f == 0) { + block.second[d] /= f; + block.first /= f; + } + + // ...and carry to the next factor. + return next_grid_factor(limits, grid, factors, + std::move(block), d, i + 1); + } + } + } + } + + /// + /// Find the divisor of the integer array \p grid that gives the + /// highest possible product not greater than \p product_limit + /// subject to the constraints given by \p coord_limit. + /// + template<typename T> + std::vector<T> + find_grid_optimal_factor(T product_limit, + const std::vector<T> &coord_limit, + const std::vector<T> &grid) { + const std::vector<std::vector<T>> factors = + map(find_integer_prime_factors<T>, grid, coord_limit); + const auto limits = std::make_pair(product_limit, coord_limit); + auto best = std::make_pair(T(1), std::vector<T>(grid.size(), T(1))); + + for (auto block = best; + block.first != 0 && best.first != product_limit; + block = detail::next_grid_factor(limits, grid, factors, block)) { + if (block.first > best.first) + best = block; + } + + return best.second; + } + } +} + +#endif diff --git a/src/gallium/frontends/clover/util/functional.hpp b/src/gallium/frontends/clover/util/functional.hpp new file mode 100644 index 00000000000..fc281c5c79a --- /dev/null +++ b/src/gallium/frontends/clover/util/functional.hpp @@ -0,0 +1,419 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_UTIL_FUNCTIONAL_HPP +#define CLOVER_UTIL_FUNCTIONAL_HPP + +#include <type_traits> + +namespace clover { + struct identity { + template<typename T> + typename std::remove_reference<T>::type + operator()(T &&x) const { + return x; + } + }; + + struct plus { + template<typename T, typename S> + typename std::common_type<T, S>::type + operator()(T x, S y) const { + return x + y; + } + }; + + struct minus { + template<typename T, typename S> + typename std::common_type<T, S>::type + operator()(T x, S y) const { + return x - y; + } + }; + + struct negate { + template<typename T> + T + operator()(T x) const { + return -x; + } + }; + + struct multiplies { + template<typename T, typename S> + typename std::common_type<T, S>::type + operator()(T x, S y) const { + return x * y; + } + }; + + struct divides { + template<typename T, typename S> + typename std::common_type<T, S>::type + operator()(T x, S y) const { + return x / y; + } + }; + + struct modulus { + template<typename T, typename S> + typename std::common_type<T, S>::type + operator()(T x, S y) const { + return x % y; + } + }; + + struct minimum { + template<typename T> + T + operator()(T x) const { + return x; + } + + template<typename T, typename... Ts> + T + operator()(T x, Ts... xs) const { + T y = minimum()(xs...); + return x < y ? x : y; + } + }; + + struct maximum { + template<typename T> + T + operator()(T x) const { + return x; + } + + template<typename T, typename... Ts> + T + operator()(T x, Ts... xs) const { + T y = maximum()(xs...); + return x < y ? y : x; + } + }; + + struct preincs { + template<typename T> + T & + operator()(T &x) const { + return ++x; + } + }; + + struct predecs { + template<typename T> + T & + operator()(T &x) const { + return --x; + } + }; + + template<typename T> + class multiplies_by_t { + public: + multiplies_by_t(T x) : x(x) { + } + + template<typename S> + typename std::common_type<T, S>::type + operator()(S y) const { + return x * y; + } + + private: + T x; + }; + + template<typename T> + multiplies_by_t<T> + multiplies_by(T x) { + return { x }; + } + + template<typename T> + class preincs_by_t { + public: + preincs_by_t(T n) : n(n) { + } + + template<typename S> + S & + operator()(S &x) const { + return x += n; + } + + private: + T n; + }; + + template<typename T> + preincs_by_t<T> + preincs_by(T n) { + return { n }; + } + + template<typename T> + class predecs_by_t { + public: + predecs_by_t(T n) : n(n) { + } + + template<typename S> + S & + operator()(S &x) const { + return x -= n; + } + + private: + T n; + }; + + template<typename T> + predecs_by_t<T> + predecs_by(T n) { + return { n }; + } + + struct greater { + template<typename T, typename S> + bool + operator()(T x, S y) const { + return x > y; + } + }; + + struct evals { + template<typename T> + auto + operator()(T &&x) const -> decltype(x()) { + return x(); + } + }; + + struct derefs { + template<typename T> + auto + operator()(T &&x) const -> decltype(*x) { + return *x; + } + }; + + struct addresses { + template<typename T> + T * + operator()(T &x) const { + return &x; + } + + template<typename T> + T * + operator()(std::reference_wrapper<T> x) const { + return &x.get(); + } + }; + + struct begins { + template<typename T> + auto + operator()(T &x) const -> decltype(x.begin()) { + return x.begin(); + } + }; + + struct ends { + template<typename T> + auto + operator()(T &x) const -> decltype(x.end()) { + return x.end(); + } + }; + + struct sizes { + template<typename T> + auto + operator()(T &x) const -> decltype(x.size()) { + return x.size(); + } + }; + + template<typename T> + class advances_by_t { + public: + advances_by_t(T n) : n(n) { + } + + template<typename S> + S + operator()(S &&it) const { + std::advance(it, n); + return it; + } + + private: + T n; + }; + + template<typename T> + advances_by_t<T> + advances_by(T n) { + return { n }; + } + + struct zips { + template<typename... Ts> + std::tuple<Ts...> + operator()(Ts &&... xs) const { + return std::tuple<Ts...>(std::forward<Ts>(xs)...); + } + }; + + struct is_zero { + template<typename T> + bool + operator()(const T &x) const { + return x == 0; + } + }; + + struct keys { + template<typename P> + auto + operator()(P &&p) const -> decltype(std::get<0>(std::forward<P>(p))) { + return std::get<0>(std::forward<P>(p)); + } + }; + + struct values { + template<typename P> + auto + operator()(P &&p) const -> decltype(std::get<1>(std::forward<P>(p))) { + return std::get<1>(std::forward<P>(p)); + } + }; + + template<typename T> + class equals_t { + public: + equals_t(T &&x) : x(x) {} + + template<typename S> + bool + operator()(S &&y) const { + return x == y; + } + + private: + T x; + }; + + template<typename T> + equals_t<T> + equals(T &&x) { + return { std::forward<T>(x) }; + } + + class name_equals { + public: + name_equals(const std::string &name) : name(name) { + } + + template<typename T> + bool + operator()(const T &x) const { + return std::string(x.name.begin(), x.name.end()) == name; + } + + private: + const std::string &name; + }; + + class id_equals { + public: + id_equals(const uint32_t id) : id(id) { + } + + template<typename T> + bool + operator()(const T &x) const { + return x.id == id; + } + + private: + const uint32_t id; + }; + + template<typename T> + class key_equals_t { + public: + key_equals_t(T &&x) : x(x) { + } + + template<typename P> + bool + operator()(const P &p) const { + return p.first == x; + } + + private: + T x; + }; + + template<typename T> + key_equals_t<T> + key_equals(T &&x) { + return { std::forward<T>(x) }; + } + + template<typename T> + class type_equals_t { + public: + type_equals_t(T type) : type(type) { + } + + template<typename S> + bool + operator()(const S &x) const { + return x.type == type; + } + + private: + T type; + }; + + template<typename T> + type_equals_t<T> + type_equals(T x) { + return { x }; + } + + struct interval_overlaps { + template<typename T> + bool + operator()(T x0, T x1, T y0, T y1) { + return ((x0 <= y0 && y0 < x1) || + (y0 <= x0 && x0 < y1)); + } + }; +} + +#endif diff --git a/src/gallium/frontends/clover/util/lazy.hpp b/src/gallium/frontends/clover/util/lazy.hpp new file mode 100644 index 00000000000..e32a8f8b1b9 --- /dev/null +++ b/src/gallium/frontends/clover/util/lazy.hpp @@ -0,0 +1,161 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_UTIL_LAZY_HPP +#define CLOVER_UTIL_LAZY_HPP + +#include <type_traits> +#include <stdexcept> +#include <memory> + +namespace clover { + namespace detail { + template<typename T> + class basic_lazy { + public: + virtual + ~basic_lazy() { + } + + virtual basic_lazy * + clone() const = 0; + + virtual + operator T() const = 0; + }; + + template<typename T, typename F> + class deferred_lazy : public basic_lazy<T> { + public: + template<typename G> + deferred_lazy(G &&f) : f(new F(std::forward<G>(f))) { + } + + virtual basic_lazy<T> * + clone() const { + return new deferred_lazy(*this); + } + + operator T() const { + if (f) { + x = (*f)(); + f = {}; + } + + return x; + } + + private: + mutable std::shared_ptr<F> f; + mutable T x; + }; + + template<typename T> + class strict_lazy : public basic_lazy<T> { + public: + template<typename S> + strict_lazy(S &&x) : x(std::forward<S>(x)) { + } + + virtual basic_lazy<T> * + clone() const { + return new strict_lazy(*this); + } + + operator T() const { + return x; + } + + private: + T x; + }; + } + + /// + /// Object that represents a value of type \a T that is calculated + /// lazily as soon as it is required. + /// + template<typename T> + class lazy { + public: + class undefined_error : std::logic_error { + public: + undefined_error() : std::logic_error("") { + } + }; + + /// + /// Initialize to some fixed value \a x which isn't calculated + /// lazily. + /// + lazy(T x) : obj(new detail::strict_lazy<T>(x)) { + } + + /// + /// Initialize by providing a functor \a f that will calculate + /// the value on-demand. + /// + template<typename F> + lazy(F &&f) : obj(new detail::deferred_lazy< + T, typename std::remove_reference<F>::type + >(std::forward<F>(f))) { + } + + /// + /// Initialize to undefined. + /// + lazy() : lazy([]() { + throw undefined_error(); + return T(); + }) { + } + + lazy(const lazy &other) : obj(obj->clone()) { + } + + lazy(lazy &&other) : obj(NULL) { + std::swap(obj, other.obj); + } + + ~lazy() { + delete obj; + } + + lazy & + operator=(lazy other) { + std::swap(obj, other.obj); + return *this; + } + + /// + /// Evaluate the value. + /// + operator T() const { + return *obj; + } + + private: + detail::basic_lazy<T> *obj; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/util/pointer.hpp b/src/gallium/frontends/clover/util/pointer.hpp new file mode 100644 index 00000000000..7bb9951aef6 --- /dev/null +++ b/src/gallium/frontends/clover/util/pointer.hpp @@ -0,0 +1,284 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_UTIL_POINTER_HPP +#define CLOVER_UTIL_POINTER_HPP + +#include <atomic> + +namespace clover { + /// + /// Some helper functions for raw pointer operations + /// + template <class T> + static bool + ptr_is_aligned(const T *ptr, uintptr_t a) noexcept { + assert(a == (a & -a)); + uintptr_t ptr_value = reinterpret_cast<uintptr_t>(ptr); + return (ptr_value & (a - 1)) == 0; + } + + /// + /// Base class for objects that support reference counting. + /// + class ref_counter { + public: + ref_counter(unsigned value = 1) : _ref_count(value) {} + + unsigned + ref_count() const { + return _ref_count; + } + + void + retain() { + _ref_count++; + } + + bool + release() { + return (--_ref_count) == 0; + } + + private: + std::atomic<unsigned> _ref_count; + }; + + /// + /// Simple reference to a clover::ref_counter object. Unlike + /// clover::intrusive_ptr and clover::intrusive_ref, it does nothing + /// special when the reference count drops to zero. + /// + class ref_holder { + public: + ref_holder(ref_counter &o) : p(&o) { + p->retain(); + } + + ref_holder(const ref_holder &ref) : + ref_holder(*ref.p) { + } + + ref_holder(ref_holder &&ref) : + p(ref.p) { + ref.p = NULL; + } + + ~ref_holder() { + if (p) + p->release(); + } + + ref_holder & + operator=(ref_holder ref) { + std::swap(ref.p, p); + return *this; + } + + bool + operator==(const ref_holder &ref) const { + return p == ref.p; + } + + bool + operator!=(const ref_holder &ref) const { + return p != ref.p; + } + + private: + ref_counter *p; + }; + + /// + /// Intrusive smart pointer for objects that implement the + /// clover::ref_counter interface. + /// + template<typename T> + class intrusive_ptr { + public: + intrusive_ptr(T *q = NULL) : p(q) { + if (p) + p->retain(); + } + + intrusive_ptr(const intrusive_ptr &ptr) : + intrusive_ptr(ptr.p) { + } + + intrusive_ptr(intrusive_ptr &&ptr) : + p(ptr.p) { + ptr.p = NULL; + } + + ~intrusive_ptr() { + if (p && p->release()) + delete p; + } + + intrusive_ptr & + operator=(intrusive_ptr ptr) { + std::swap(ptr.p, p); + return *this; + } + + bool + operator==(const intrusive_ptr &ref) const { + return p == ref.p; + } + + bool + operator!=(const intrusive_ptr &ref) const { + return p != ref.p; + } + + T & + operator*() const { + return *p; + } + + T * + operator->() const { + return p; + } + + T * + operator()() const { + return p; + } + + explicit operator bool() const { + return p; + } + + explicit operator T *() const { + return p; + } + + private: + T *p; + }; + + /// + /// Intrusive smart reference for objects that implement the + /// clover::ref_counter interface. + /// + template<typename T> + class intrusive_ref { + public: + intrusive_ref(T &o) : p(&o) { + p->retain(); + } + + intrusive_ref(const intrusive_ref &ref) : + intrusive_ref(*ref.p) { + } + + intrusive_ref(intrusive_ref &&ref) : + p(ref.p) { + ref.p = NULL; + } + + ~intrusive_ref() { + if (p && p->release()) + delete p; + } + + intrusive_ref & + operator=(intrusive_ref ref) { + std::swap(ref.p, p); + return *this; + } + + bool + operator==(const intrusive_ref &ref) const { + return p == ref.p; + } + + bool + operator!=(const intrusive_ref &ref) const { + return p != ref.p; + } + + T & + operator()() const { + return *p; + } + + operator T &() const { + return *p; + } + + private: + T *p; + }; + + /// + /// Initialize a clover::intrusive_ref from a newly created object + /// using the specified constructor arguments. + /// + template<typename T, typename... As> + intrusive_ref<T> + create(As &&... as) { + intrusive_ref<T> ref { *new T(std::forward<As>(as)...) }; + ref().release(); + return ref; + } + + /// + /// Class that implements the usual pointer interface but in fact + /// contains the object it seems to be pointing to. + /// + template<typename T> + class pseudo_ptr { + public: + pseudo_ptr(T x) : x(x) { + } + + pseudo_ptr(const pseudo_ptr &p) : x(p.x) { + } + + pseudo_ptr & + operator=(const pseudo_ptr &p) { + x = p.x; + return *this; + } + + T & + operator*() { + return x; + } + + T * + operator->() { + return &x; + } + + explicit operator bool() const { + return true; + } + + private: + T x; + }; +} + +#endif diff --git a/src/gallium/frontends/clover/util/range.hpp b/src/gallium/frontends/clover/util/range.hpp new file mode 100644 index 00000000000..b082359ee86 --- /dev/null +++ b/src/gallium/frontends/clover/util/range.hpp @@ -0,0 +1,419 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_UTIL_RANGE_HPP +#define CLOVER_UTIL_RANGE_HPP + +#include <array> +#include <vector> + +#include "util/adaptor.hpp" + +namespace clover { + /// + /// Class that identifies container types where the elements of a + /// range can be stored by the type conversion operator. + /// + /// \a T identifies the range element type. + /// + template<typename T, typename V> + struct range_store_traits; + + template<typename T, typename S> + struct range_store_traits<T, std::vector<S>> { + typedef void enable; + + template<typename R> + static std::vector<S> + create(const R &r) { + return { r.begin(), r.end() }; + } + }; + + template<typename T, typename S, std::size_t N> + struct range_store_traits<T, std::array<S, N>> { + typedef void enable; + + template<typename R> + static std::array<S, N> + create(const R &r) { + std::array<S, N> v; + assert(r.size() == v.size()); + copy(r, v.begin()); + return v; + } + }; + + namespace detail { + /// + /// Common functionality that is shared by other implementations + /// of the container concept. + /// + template<typename R, typename I, typename CI> + class basic_range { + public: + typedef I iterator; + typedef CI const_iterator; + typedef typename std::iterator_traits<iterator>::value_type value_type; + typedef typename std::iterator_traits<iterator>::reference + reference; + typedef typename std::iterator_traits<const_iterator>::reference + const_reference; + typedef typename std::iterator_traits<iterator>::difference_type + difference_type; + typedef std::size_t size_type; + + bool + operator==(const basic_range &r) const { + return *static_cast<const R *>(this) == r; + } + + bool + operator!=(const basic_range &r) const { + return !(*this == r); + } + + iterator + begin() { + return static_cast<R *>(this)->begin(); + } + + iterator + end() { + return static_cast<R *>(this)->end(); + } + + const_iterator + begin() const { + return static_cast<const R *>(this)->begin(); + } + + const_iterator + end() const { + return static_cast<const R *>(this)->end(); + } + + std::reverse_iterator<iterator> + rbegin() { + return { begin() }; + } + + std::reverse_iterator<iterator> + rend() { + return { end() }; + } + + reference + front() { + return *begin(); + } + + reference + back() { + return *(end() - 1); + } + + bool + empty() const { + return begin() == end(); + } + + reference + at(size_type i) { + if (i >= static_cast<const R *>(this)->size()) + throw std::out_of_range(""); + + return begin()[i]; + } + + const_reference + at(size_type i) const { + if (i >= static_cast<const R *>(this)->size()) + throw std::out_of_range(""); + + return begin()[i]; + } + + reference + operator[](size_type i) { + return begin()[i]; + } + + const_reference + operator[](size_type i) const { + return begin()[i]; + } + + template<typename V> + using store_traits = range_store_traits< + typename std::remove_cv<value_type>::type, V + >; + + template<typename V, + typename = typename store_traits<V>::enable> + operator V() const { + return store_traits<V>::create(*static_cast<const R *>(this)); + } + }; + } + + /// + /// Range that contains all elements delimited by an iterator pair + /// (\a i, \a j). Use range() as convenience constructor. + /// + template<typename I> + class iterator_range : public detail::basic_range<iterator_range<I>, I, I> { + public: + typedef detail::basic_range<iterator_range<I>, I, I> super; + + iterator_range() : i(), j() { + } + + iterator_range(I i, I j) : i(i), j(j) { + } + + bool + operator==(const iterator_range &r) const { + return i == r.i && j == r.j; + } + + I + begin() const { + return i; + } + + I + end() const { + return j; + } + + typename super::size_type + size() const { + return end() - begin(); + } + + private: + I i, j; + }; + + namespace detail { + template<typename T> + using preferred_iterator_type = decltype(std::declval<T>().begin()); + } + + /// + /// Range that transforms the contents of a number of source ranges + /// \a os element-wise by using the provided functor \a f. Use + /// map() as convenience constructor. + /// + template<typename F, typename... Os> + class adaptor_range : + public detail::basic_range<adaptor_range<F, Os...>, + detail::iterator_adaptor< + F, detail::preferred_iterator_type<Os>...>, + detail::iterator_adaptor< + F, detail::preferred_iterator_type<const Os>...> + > { + public: + typedef detail::basic_range<adaptor_range<F, Os...>, + detail::iterator_adaptor< + F, detail::preferred_iterator_type<Os>...>, + detail::iterator_adaptor< + F, detail::preferred_iterator_type<const Os>...> + > super; + + template<typename G, typename... Rs> + adaptor_range(G &&f, Rs &&... os) : + f(std::forward<G>(f)), os(std::forward<Rs>(os)...) { + } + + bool + operator==(const adaptor_range &r) const { + return f == r.f && os == r.os; + } + + typename super::iterator + begin() { + return { f, tuple::map(begins(), os) }; + } + + typename super::iterator + end() { + return { f, tuple::map(advances_by(size()), + tuple::map(begins(), os)) }; + } + + typename super::const_iterator + begin() const { + return { f, tuple::map(begins(), os) }; + } + + typename super::const_iterator + end() const { + return { f, tuple::map(advances_by(size()), + tuple::map(begins(), os)) }; + } + + typename super::size_type + size() const { + return tuple::apply(minimum(), tuple::map(sizes(), os)); + } + + private: + F f; + std::tuple<Os...> os; + }; + + /// + /// Range that contains all elements delimited by the index pair + /// (\a i, \a j) in the source range \a r. Use slice() as + /// convenience constructor. + /// + template<typename O> + class slice_range : + public detail::basic_range<slice_range<O>, + detail::preferred_iterator_type<O>, + detail::preferred_iterator_type<const O>> { + public: + typedef detail::basic_range<slice_range<O>, + detail::preferred_iterator_type<O>, + detail::preferred_iterator_type<const O> + > super; + + template<typename R> + slice_range(R &&r, typename super::size_type i, + typename super::size_type j) : + o(std::forward<R>(r)), i(i), j(j) { + } + + bool + operator==(const slice_range &r) const { + return o == r.o && i == r.i && j == r.j; + } + + typename super::iterator + begin() { + return std::next(o.begin(), i); + } + + typename super::iterator + end() { + return std::next(o.begin(), j); + } + + typename super::const_iterator + begin() const { + return std::next(o.begin(), i); + } + + typename super::const_iterator + end() const { + return std::next(o.begin(), j); + } + + typename super::size_type + size() const { + return j - i; + } + + private: + O o; + typename super::size_type i, j; + }; + + /// + /// Create a range from an iterator pair (\a i, \a j). + /// + /// \sa iterator_range. + /// + template<typename T> + iterator_range<T> + range(T i, T j) { + return { i, j }; + } + + /// + /// Create a range of \a n elements starting from iterator \a i. + /// + /// \sa iterator_range. + /// + template<typename T> + iterator_range<T> + range(T i, typename std::iterator_traits<T>::difference_type n) { + return { i, i + n }; + } + + /// + /// Create a range by transforming the contents of a number of + /// source ranges \a rs element-wise using a provided functor \a f. + /// + /// \sa adaptor_range. + /// + template<typename F, typename... Rs> + adaptor_range<F, Rs...> + map(F &&f, Rs &&... rs) { + return { std::forward<F>(f), std::forward<Rs>(rs)... }; + } + + /// + /// Create a range identical to another range \a r. + /// + template<typename R> + adaptor_range<identity, R> + range(R &&r) { + return { identity(), std::forward<R>(r) }; + } + + /// + /// Create a range by taking the elements delimited by the index + /// pair (\a i, \a j) in a source range \a r. + /// + /// \sa slice_range. + /// + template<typename R> + slice_range<R> + slice(R &&r, typename slice_range<R>::size_type i, + typename slice_range<R>::size_type j) { + return { std::forward<R>(r), i, j }; + } + + /// + /// Range that behaves as a vector of references of type \a T. + /// + /// Useful because STL containers cannot contain references to + /// objects as elements. + /// + template<typename T> + class ref_vector : public adaptor_range<derefs, std::vector<T *>> { + public: + ref_vector(std::initializer_list<std::reference_wrapper<T>> il) : + adaptor_range<derefs, std::vector<T *>>(derefs(), map(addresses(), il)) { + } + + template<typename R> + ref_vector(R &&r) : adaptor_range<derefs, std::vector<T *>>( + derefs(), map(addresses(), std::forward<R>(r))) { + } + }; +} + +#endif diff --git a/src/gallium/frontends/clover/util/tuple.hpp b/src/gallium/frontends/clover/util/tuple.hpp new file mode 100644 index 00000000000..bd49684a314 --- /dev/null +++ b/src/gallium/frontends/clover/util/tuple.hpp @@ -0,0 +1,117 @@ +// +// Copyright 2013 Francisco Jerez +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +// THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR +// OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +// OTHER DEALINGS IN THE SOFTWARE. +// + +#ifndef CLOVER_UTIL_TUPLE_HPP +#define CLOVER_UTIL_TUPLE_HPP + +#include <tuple> + +namespace clover { + namespace tuple { + /// + /// Static sequence of integers. + /// + template<int... Is> + struct integral_sequence; + + /// + /// Static sequence containing all integers from 0 to N-1. + /// + template<int N, int... Is> + struct enumerate { + typedef typename enumerate<N-1, N-1, Is...>::type + type; + }; + + template<int... Is> + struct enumerate<0, Is...> { + typedef integral_sequence<Is...> type; + }; + + namespace detail { + template<typename F, typename T, + typename E = typename enumerate<std::tuple_size< + typename std::remove_reference<T>::type>::value + >::type> + struct _apply; + + template<typename F, typename T, int... Is> + struct _apply<F, T, integral_sequence<Is...>> { + typedef typename std::remove_reference<F>::type func_type; + typedef decltype( + std::declval<func_type>()(std::get<Is>(std::declval<T &&>())...) + ) value_type; + + static value_type + eval(F &&f, T &&t) { + return f(std::get<Is>(std::forward<T>(t))...); + } + }; + } + + /// + /// Evaluate function \a f with the elements of tuple \a t + /// expanded as arguments. + /// + template<typename F, typename T> + typename detail::_apply<F, T>::value_type + apply(F &&f, T &&t) { + return detail::_apply<F, T>::eval(std::forward<F>(f), + std::forward<T>(t)); + } + + namespace detail { + template<typename F, typename T, + typename E = typename enumerate<std::tuple_size< + typename std::remove_reference<T>::type>::value + >::type> + struct _map; + + template<typename F, typename T, int... Is> + struct _map<F, T, integral_sequence<Is...>> { + typedef typename std::remove_reference<F>::type func_type; + typedef std::tuple< + decltype(std::declval<func_type>()( + std::get<Is>(std::declval<T &&>())))... + > value_type; + + static value_type + eval(F &&f, T &&t) { + return value_type(f(std::get<Is>(std::forward<T>(t)))...); + } + }; + } + + /// + /// Evaluate function \a f on each element of the tuple \a t and + /// return the resulting values as a new tuple. + /// + template<typename F, typename T> + typename detail::_map<F, T>::value_type + map(F &&f, T &&t) { + return detail::_map<F, T>::eval(std::forward<F>(f), + std::forward<T>(t)); + } + } +} + +#endif |