//===- OCLUtil.cpp - OCL Utilities ----------------------------------------===//
//
//                     The LLVM/SPIRV Translator
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
// Copyright (c) 2014 Advanced Micro Devices, Inc. All rights reserved.
//
// Permission is hereby granted, free of charge, to any person obtaining a
// copy of this software and associated documentation files (the "Software"),
// to deal with the Software without restriction, including without limitation
// the rights to use, copy, modify, merge, publish, distribute, sublicense,
// and/or sell copies of the Software, and to permit persons to whom the
// Software is furnished to do so, subject to the following conditions:
//
// Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimers.
// Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimers in the documentation
// and/or other materials provided with the distribution.
// Neither the names of Advanced Micro Devices, Inc., nor the names of its
// contributors may be used to endorse or promote products derived from this
// Software without specific prior written permission.
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH
// THE SOFTWARE.
//
//===----------------------------------------------------------------------===//
//
// This file implements OCL utility functions.
//
//===----------------------------------------------------------------------===//

#include "OCLUtil.h"
#include "SPIRVEntry.h"
#include "SPIRVFunction.h"
#include "SPIRVInstruction.h"
#include "SPIRVInternal.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstVisitor.h"
#include "llvm/IR/Instructions.h"
#include "llvm/Pass.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"

#define DEBUG_TYPE "oclutil"

using namespace llvm;
using namespace SPIRV;

namespace OCLUtil {

#ifndef SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#define SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE SPIRAS_Private
#endif

#ifndef SPIRV_QUEUE_T_ADDR_SPACE
#define SPIRV_QUEUE_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif

#ifndef SPIRV_EVENT_T_ADDR_SPACE
#define SPIRV_EVENT_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif

#ifndef SPIRV_AVC_INTEL_T_ADDR_SPACE
#define SPIRV_AVC_INTEL_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif

#ifndef SPIRV_CLK_EVENT_T_ADDR_SPACE
#define SPIRV_CLK_EVENT_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif

#ifndef SPIRV_SAMPLER_T_ADDR_SPACE
#define SPIRV_SAMPLER_T_ADDR_SPACE SPIRAS_Constant
#endif

#ifndef SPIRV_RESERVE_ID_T_ADDR_SPACE
#define SPIRV_RESERVE_ID_T_ADDR_SPACE SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE
#endif
// Excerpt from SPIR 2.0 spec.:
//   Pipe objects are represented using pointers to the opaque %opencl.pipe LLVM
//   structure type which reside in the global address space.
#ifndef SPIRV_PIPE_ADDR_SPACE
#define SPIRV_PIPE_ADDR_SPACE SPIRAS_Global
#endif
// Excerpt from SPIR 2.0 spec.:
//   Note: Images data types reside in global memory and hence should be marked
//   as such in the "kernel arg addr space" metadata.
#ifndef SPIRV_IMAGE_ADDR_SPACE
#define SPIRV_IMAGE_ADDR_SPACE SPIRAS_Global
#endif

} // namespace OCLUtil

///////////////////////////////////////////////////////////////////////////////
//
// Map definitions
//
///////////////////////////////////////////////////////////////////////////////

using namespace OCLUtil;
namespace SPIRV {

template <> void SPIRVMap<OCLMemFenceKind, MemorySemanticsMask>::init() {
  add(OCLMF_Local, MemorySemanticsWorkgroupMemoryMask);
  add(OCLMF_Global, MemorySemanticsCrossWorkgroupMemoryMask);
  add(OCLMF_Image, MemorySemanticsImageMemoryMask);
}

template <>
void SPIRVMap<OCLMemFenceExtendedKind, MemorySemanticsMask>::init() {
  add(OCLMFEx_Local, MemorySemanticsWorkgroupMemoryMask);
  add(OCLMFEx_Global, MemorySemanticsCrossWorkgroupMemoryMask);
  add(OCLMFEx_Local_Global, MemorySemanticsWorkgroupMemoryMask |
                                MemorySemanticsCrossWorkgroupMemoryMask);
  add(OCLMFEx_Image, MemorySemanticsImageMemoryMask);
  add(OCLMFEx_Image_Local,
      MemorySemanticsWorkgroupMemoryMask | MemorySemanticsImageMemoryMask);
  add(OCLMFEx_Image_Global,
      MemorySemanticsCrossWorkgroupMemoryMask | MemorySemanticsImageMemoryMask);
  add(OCLMFEx_Image_Local_Global, MemorySemanticsWorkgroupMemoryMask |
                                      MemorySemanticsCrossWorkgroupMemoryMask |
                                      MemorySemanticsImageMemoryMask);
}

template <>
void SPIRVMap<OCLMemOrderKind, unsigned, MemorySemanticsMask>::init() {
  add(OCLMO_relaxed, MemorySemanticsMaskNone);
  add(OCLMO_acquire, MemorySemanticsAcquireMask);
  add(OCLMO_release, MemorySemanticsReleaseMask);
  add(OCLMO_acq_rel, MemorySemanticsAcquireReleaseMask);
  add(OCLMO_seq_cst, MemorySemanticsSequentiallyConsistentMask);
}

template <> void SPIRVMap<OCLScopeKind, Scope>::init() {
  add(OCLMS_work_item, ScopeInvocation);
  add(OCLMS_work_group, ScopeWorkgroup);
  add(OCLMS_device, ScopeDevice);
  add(OCLMS_all_svm_devices, ScopeCrossDevice);
  add(OCLMS_sub_group, ScopeSubgroup);
}

template <> void SPIRVMap<std::string, Scope>::init() {
  add("work_item", ScopeInvocation);
  add("workgroup", ScopeWorkgroup);
  add("device", ScopeDevice);
  add("all_svm_devices", ScopeCrossDevice);
  add("subgroup", ScopeSubgroup);
}

template <> void SPIRVMap<std::string, SPIRVGroupOperationKind>::init() {
  add("reduce", GroupOperationReduce);
  add("scan_inclusive", GroupOperationInclusiveScan);
  add("scan_exclusive", GroupOperationExclusiveScan);
  add("ballot_bit_count", GroupOperationReduce);
  add("ballot_inclusive_scan", GroupOperationInclusiveScan);
  add("ballot_exclusive_scan", GroupOperationExclusiveScan);
  add("non_uniform_reduce", GroupOperationReduce);
  add("non_uniform_scan_inclusive", GroupOperationInclusiveScan);
  add("non_uniform_scan_exclusive", GroupOperationExclusiveScan);
  add("non_uniform_reduce_logical", GroupOperationReduce);
  add("non_uniform_scan_inclusive_logical", GroupOperationInclusiveScan);
  add("non_uniform_scan_exclusive_logical", GroupOperationExclusiveScan);
  add("clustered_reduce", GroupOperationClusteredReduce);
}

template <> void SPIRVMap<std::string, SPIRVFPRoundingModeKind>::init() {
  add("rte", FPRoundingModeRTE);
  add("rtz", FPRoundingModeRTZ);
  add("rtp", FPRoundingModeRTP);
  add("rtn", FPRoundingModeRTN);
}

template <> void SPIRVMap<OclExt::Kind, std::string>::init() {
#define _SPIRV_OP(x) add(OclExt::x, #x);
  _SPIRV_OP(cl_images)
  _SPIRV_OP(cl_doubles)
  _SPIRV_OP(cl_khr_int64_base_atomics)
  _SPIRV_OP(cl_khr_int64_extended_atomics)
  _SPIRV_OP(cl_khr_fp16)
  _SPIRV_OP(cl_khr_gl_sharing)
  _SPIRV_OP(cl_khr_gl_event)
  _SPIRV_OP(cl_khr_d3d10_sharing)
  _SPIRV_OP(cl_khr_media_sharing)
  _SPIRV_OP(cl_khr_d3d11_sharing)
  _SPIRV_OP(cl_khr_global_int32_base_atomics)
  _SPIRV_OP(cl_khr_global_int32_extended_atomics)
  _SPIRV_OP(cl_khr_local_int32_base_atomics)
  _SPIRV_OP(cl_khr_local_int32_extended_atomics)
  _SPIRV_OP(cl_khr_byte_addressable_store)
  _SPIRV_OP(cl_khr_3d_image_writes)
  _SPIRV_OP(cl_khr_gl_msaa_sharing)
  _SPIRV_OP(cl_khr_depth_images)
  _SPIRV_OP(cl_khr_gl_depth_images)
  _SPIRV_OP(cl_khr_subgroups)
  _SPIRV_OP(cl_khr_mipmap_image)
  _SPIRV_OP(cl_khr_mipmap_image_writes)
  _SPIRV_OP(cl_khr_egl_event)
  _SPIRV_OP(cl_khr_srgb_image_writes)
  _SPIRV_OP(cl_khr_extended_bit_ops)
#undef _SPIRV_OP
}

template <> void SPIRVMap<OclExt::Kind, SPIRVCapabilityKind>::init() {
  add(OclExt::cl_images, CapabilityImageBasic);
  add(OclExt::cl_doubles, CapabilityFloat64);
  add(OclExt::cl_khr_int64_base_atomics, CapabilityInt64Atomics);
  add(OclExt::cl_khr_int64_extended_atomics, CapabilityInt64Atomics);
  add(OclExt::cl_khr_fp16, CapabilityFloat16);
  add(OclExt::cl_khr_subgroups, CapabilityGroups);
  add(OclExt::cl_khr_mipmap_image, CapabilityImageMipmap);
  add(OclExt::cl_khr_mipmap_image_writes, CapabilityImageMipmap);
  add(OclExt::cl_khr_extended_bit_ops, CapabilityBitInstructions);
}

/// Map OpenCL work functions to SPIR-V builtin variables.
template <> void SPIRVMap<std::string, SPIRVBuiltinVariableKind>::init() {
  add("get_work_dim", BuiltInWorkDim);
  add("get_global_size", BuiltInGlobalSize);
  add("get_global_id", BuiltInGlobalInvocationId);
  add("get_global_offset", BuiltInGlobalOffset);
  add("get_local_size", BuiltInWorkgroupSize);
  add("get_enqueued_local_size", BuiltInEnqueuedWorkgroupSize);
  add("get_local_id", BuiltInLocalInvocationId);
  add("get_num_groups", BuiltInNumWorkgroups);
  add("get_group_id", BuiltInWorkgroupId);
  add("get_global_linear_id", BuiltInGlobalLinearId);
  add("get_local_linear_id", BuiltInLocalInvocationIndex);
  // cl_khr_subgroups
  add("get_sub_group_size", BuiltInSubgroupSize);
  add("get_max_sub_group_size", BuiltInSubgroupMaxSize);
  add("get_num_sub_groups", BuiltInNumSubgroups);
  add("get_enqueued_num_sub_groups", BuiltInNumEnqueuedSubgroups);
  add("get_sub_group_id", BuiltInSubgroupId);
  add("get_sub_group_local_id", BuiltInSubgroupLocalInvocationId);
  // cl_khr_subgroup_ballot
  add("get_sub_group_eq_mask", BuiltInSubgroupEqMask);
  add("get_sub_group_ge_mask", BuiltInSubgroupGeMask);
  add("get_sub_group_gt_mask", BuiltInSubgroupGtMask);
  add("get_sub_group_le_mask", BuiltInSubgroupLeMask);
  add("get_sub_group_lt_mask", BuiltInSubgroupLtMask);
}

// Maps uniqued OCL builtin function name to SPIR-V op code.
// A uniqued OCL builtin function name may be different from the real
// OCL builtin function name. e.g. instead of atomic_min, atomic_umin
// is used for atomic_min with unsigned integer parameter.
// work_group_ and sub_group_ functions are unified as group_ functions
// except work_group_barrier.
class SPIRVInstruction;
template <> void SPIRVMap<std::string, Op, SPIRVInstruction>::init() {
#define _SPIRV_OP(x, y) add("atom_" #x, OpAtomic##y);
  // cl_khr_int64_base_atomics builtins
  _SPIRV_OP(add, IAdd)
  _SPIRV_OP(sub, ISub)
  _SPIRV_OP(xchg, Exchange)
  _SPIRV_OP(dec, IDecrement)
  _SPIRV_OP(inc, IIncrement)
  _SPIRV_OP(cmpxchg, CompareExchange)
  // cl_khr_int64_extended_atomics builtins
  _SPIRV_OP(min, SMin)
  _SPIRV_OP(max, SMax)
  _SPIRV_OP(and, And)
  _SPIRV_OP(or, Or)
  _SPIRV_OP(xor, Xor)
#undef _SPIRV_OP
#define _SPIRV_OP(x, y) add("atomic_" #x, Op##y);
  // CL 2.0 atomic builtins
  _SPIRV_OP(flag_test_and_set_explicit, AtomicFlagTestAndSet)
  _SPIRV_OP(flag_clear_explicit, AtomicFlagClear)
  _SPIRV_OP(load_explicit, AtomicLoad)
  _SPIRV_OP(store_explicit, AtomicStore)
  _SPIRV_OP(exchange_explicit, AtomicExchange)
  _SPIRV_OP(compare_exchange_strong_explicit, AtomicCompareExchange)
  _SPIRV_OP(compare_exchange_weak_explicit, AtomicCompareExchangeWeak)
  _SPIRV_OP(inc, AtomicIIncrement)
  _SPIRV_OP(dec, AtomicIDecrement)
  _SPIRV_OP(fetch_add_explicit, AtomicIAdd)
  _SPIRV_OP(fetch_sub_explicit, AtomicISub)
  _SPIRV_OP(fetch_umin_explicit, AtomicUMin)
  _SPIRV_OP(fetch_umax_explicit, AtomicUMax)
  _SPIRV_OP(fetch_min_explicit, AtomicSMin)
  _SPIRV_OP(fetch_max_explicit, AtomicSMax)
  _SPIRV_OP(fetch_and_explicit, AtomicAnd)
  _SPIRV_OP(fetch_or_explicit, AtomicOr)
  _SPIRV_OP(fetch_xor_explicit, AtomicXor)
#undef _SPIRV_OP
#define _SPIRV_OP(x, y) add(#x, Op##y);
  _SPIRV_OP(dot, Dot)
  _SPIRV_OP(async_work_group_copy, GroupAsyncCopy)
  _SPIRV_OP(async_work_group_strided_copy, GroupAsyncCopy)
  _SPIRV_OP(wait_group_events, GroupWaitEvents)
  _SPIRV_OP(isequal, FOrdEqual)
  _SPIRV_OP(isnotequal, FUnordNotEqual)
  _SPIRV_OP(isgreater, FOrdGreaterThan)
  _SPIRV_OP(isgreaterequal, FOrdGreaterThanEqual)
  _SPIRV_OP(isless, FOrdLessThan)
  _SPIRV_OP(islessequal, FOrdLessThanEqual)
  _SPIRV_OP(islessgreater, FOrdNotEqual)
  _SPIRV_OP(isordered, Ordered)
  _SPIRV_OP(isunordered, Unordered)
  _SPIRV_OP(isfinite, IsFinite)
  _SPIRV_OP(isinf, IsInf)
  _SPIRV_OP(isnan, IsNan)
  _SPIRV_OP(isnormal, IsNormal)
  _SPIRV_OP(signbit, SignBitSet)
  _SPIRV_OP(any, Any)
  _SPIRV_OP(all, All)
  _SPIRV_OP(popcount, BitCount)
  _SPIRV_OP(get_fence, GenericPtrMemSemantics)
  // CL 2.0 kernel enqueue builtins
  _SPIRV_OP(enqueue_marker, EnqueueMarker)
  _SPIRV_OP(enqueue_kernel, EnqueueKernel)
  _SPIRV_OP(get_kernel_sub_group_count_for_ndrange_impl,
            GetKernelNDrangeSubGroupCount)
  _SPIRV_OP(get_kernel_max_sub_group_size_for_ndrange_impl,
            GetKernelNDrangeMaxSubGroupSize)
  _SPIRV_OP(get_kernel_work_group_size_impl, GetKernelWorkGroupSize)
  _SPIRV_OP(get_kernel_preferred_work_group_size_multiple_impl,
            GetKernelPreferredWorkGroupSizeMultiple)
  _SPIRV_OP(retain_event, RetainEvent)
  _SPIRV_OP(release_event, ReleaseEvent)
  _SPIRV_OP(create_user_event, CreateUserEvent)
  _SPIRV_OP(is_valid_event, IsValidEvent)
  _SPIRV_OP(set_user_event_status, SetUserEventStatus)
  _SPIRV_OP(capture_event_profiling_info, CaptureEventProfilingInfo)
  _SPIRV_OP(get_default_queue, GetDefaultQueue)
  _SPIRV_OP(ndrange_1D, BuildNDRange)
  _SPIRV_OP(ndrange_2D, BuildNDRange)
  _SPIRV_OP(ndrange_3D, BuildNDRange)
  // Generic Address Space Casts
  _SPIRV_OP(to_global, GenericCastToPtrExplicit)
  _SPIRV_OP(to_local, GenericCastToPtrExplicit)
  _SPIRV_OP(to_private, GenericCastToPtrExplicit)
  // CL 2.0 pipe builtins
  _SPIRV_OP(read_pipe_2, ReadPipe)
  _SPIRV_OP(write_pipe_2, WritePipe)
  _SPIRV_OP(read_pipe_2_bl, ReadPipeBlockingINTEL)
  _SPIRV_OP(write_pipe_2_bl, WritePipeBlockingINTEL)
  _SPIRV_OP(read_pipe_4, ReservedReadPipe)
  _SPIRV_OP(write_pipe_4, ReservedWritePipe)
  _SPIRV_OP(reserve_read_pipe, ReserveReadPipePackets)
  _SPIRV_OP(reserve_write_pipe, ReserveWritePipePackets)
  _SPIRV_OP(commit_read_pipe, CommitReadPipe)
  _SPIRV_OP(commit_write_pipe, CommitWritePipe)
  _SPIRV_OP(is_valid_reserve_id, IsValidReserveId)
  _SPIRV_OP(group_reserve_read_pipe, GroupReserveReadPipePackets)
  _SPIRV_OP(group_reserve_write_pipe, GroupReserveWritePipePackets)
  _SPIRV_OP(group_commit_read_pipe, GroupCommitReadPipe)
  _SPIRV_OP(group_commit_write_pipe, GroupCommitWritePipe)
  _SPIRV_OP(get_pipe_num_packets_ro, GetNumPipePackets)
  _SPIRV_OP(get_pipe_num_packets_wo, GetNumPipePackets)
  _SPIRV_OP(get_pipe_max_packets_ro, GetMaxPipePackets)
  _SPIRV_OP(get_pipe_max_packets_wo, GetMaxPipePackets)
  // CL 2.0 workgroup builtins
  _SPIRV_OP(group_all, GroupAll)
  _SPIRV_OP(group_any, GroupAny)
  _SPIRV_OP(group_broadcast, GroupBroadcast)
  _SPIRV_OP(group_iadd, GroupIAdd)
  _SPIRV_OP(group_fadd, GroupFAdd)
  _SPIRV_OP(group_fmin, GroupFMin)
  _SPIRV_OP(group_umin, GroupUMin)
  _SPIRV_OP(group_smin, GroupSMin)
  _SPIRV_OP(group_fmax, GroupFMax)
  _SPIRV_OP(group_umax, GroupUMax)
  _SPIRV_OP(group_smax, GroupSMax)
  _SPIRV_OP(group_imul, GroupIMulKHR)
  _SPIRV_OP(group_fmul, GroupFMulKHR)
  _SPIRV_OP(group_ibitwise_and, GroupBitwiseAndKHR)
  _SPIRV_OP(group_ibitwise_or, GroupBitwiseOrKHR)
  _SPIRV_OP(group_ibitwise_xor, GroupBitwiseXorKHR)
  _SPIRV_OP(group_ilogical_and, GroupLogicalAndKHR)
  _SPIRV_OP(group_ilogical_or, GroupLogicalOrKHR)
  _SPIRV_OP(group_ilogical_xor, GroupLogicalXorKHR)
  // CL image builtins
  _SPIRV_OP(SampledImage, SampledImage)
  _SPIRV_OP(ImageSampleExplicitLod, ImageSampleExplicitLod)
  _SPIRV_OP(read_image, ImageRead)
  _SPIRV_OP(write_image, ImageWrite)
  _SPIRV_OP(get_image_channel_data_type, ImageQueryFormat)
  _SPIRV_OP(get_image_channel_order, ImageQueryOrder)
  _SPIRV_OP(get_image_num_mip_levels, ImageQueryLevels)
  _SPIRV_OP(get_image_num_samples, ImageQuerySamples)
  // Intel Subgroups builtins
  _SPIRV_OP(intel_sub_group_shuffle, SubgroupShuffleINTEL)
  _SPIRV_OP(intel_sub_group_shuffle_down, SubgroupShuffleDownINTEL)
  _SPIRV_OP(intel_sub_group_shuffle_up, SubgroupShuffleUpINTEL)
  _SPIRV_OP(intel_sub_group_shuffle_xor, SubgroupShuffleXorINTEL)
  // Intel media_block_io builtins
  _SPIRV_OP(intel_sub_group_media_block_read, SubgroupImageMediaBlockReadINTEL)
  _SPIRV_OP(intel_sub_group_media_block_write,
            SubgroupImageMediaBlockWriteINTEL)
  // cl_khr_subgroup_non_uniform_vote
  _SPIRV_OP(group_elect, GroupNonUniformElect)
  _SPIRV_OP(group_non_uniform_all, GroupNonUniformAll)
  _SPIRV_OP(group_non_uniform_any, GroupNonUniformAny)
  _SPIRV_OP(group_non_uniform_all_equal, GroupNonUniformAllEqual)
  // cl_khr_subgroup_ballot
  _SPIRV_OP(group_non_uniform_broadcast, GroupNonUniformBroadcast)
  _SPIRV_OP(group_broadcast_first, GroupNonUniformBroadcastFirst)
  _SPIRV_OP(group_ballot, GroupNonUniformBallot)
  _SPIRV_OP(group_inverse_ballot, GroupNonUniformInverseBallot)
  _SPIRV_OP(group_ballot_bit_extract, GroupNonUniformBallotBitExtract)
  _SPIRV_OP(group_ballot_bit_count_iadd, GroupNonUniformBallotBitCount)
  _SPIRV_OP(group_ballot_find_lsb, GroupNonUniformBallotFindLSB)
  _SPIRV_OP(group_ballot_find_msb, GroupNonUniformBallotFindMSB)
  // cl_khr_subgroup_non_uniform_arithmetic
  _SPIRV_OP(group_non_uniform_iadd, GroupNonUniformIAdd)
  _SPIRV_OP(group_non_uniform_fadd, GroupNonUniformFAdd)
  _SPIRV_OP(group_non_uniform_imul, GroupNonUniformIMul)
  _SPIRV_OP(group_non_uniform_fmul, GroupNonUniformFMul)
  _SPIRV_OP(group_non_uniform_smin, GroupNonUniformSMin)
  _SPIRV_OP(group_non_uniform_umin, GroupNonUniformUMin)
  _SPIRV_OP(group_non_uniform_fmin, GroupNonUniformFMin)
  _SPIRV_OP(group_non_uniform_smax, GroupNonUniformSMax)
  _SPIRV_OP(group_non_uniform_umax, GroupNonUniformUMax)
  _SPIRV_OP(group_non_uniform_fmax, GroupNonUniformFMax)
  _SPIRV_OP(group_non_uniform_iand, GroupNonUniformBitwiseAnd)
  _SPIRV_OP(group_non_uniform_ior, GroupNonUniformBitwiseOr)
  _SPIRV_OP(group_non_uniform_ixor, GroupNonUniformBitwiseXor)
  _SPIRV_OP(group_non_uniform_logical_iand, GroupNonUniformLogicalAnd)
  _SPIRV_OP(group_non_uniform_logical_ior, GroupNonUniformLogicalOr)
  _SPIRV_OP(group_non_uniform_logical_ixor, GroupNonUniformLogicalXor)
  // cl_khr_subgroup_shuffle
  _SPIRV_OP(group_shuffle, GroupNonUniformShuffle)
  _SPIRV_OP(group_shuffle_xor, GroupNonUniformShuffleXor)
  // cl_khr_subgroup_shuffle_relative
  _SPIRV_OP(group_shuffle_up, GroupNonUniformShuffleUp)
  _SPIRV_OP(group_shuffle_down, GroupNonUniformShuffleDown)
  // cl_khr_subgroup_rotate
  _SPIRV_OP(group_rotate, GroupNonUniformRotateKHR)
  _SPIRV_OP(group_clustered_rotate, GroupNonUniformRotateKHR)
  // cl_khr_extended_bit_ops
  _SPIRV_OP(bitfield_insert, BitFieldInsert)
  _SPIRV_OP(bitfield_extract_signed, BitFieldSExtract)
  _SPIRV_OP(bitfield_extract_unsigned, BitFieldUExtract)
  _SPIRV_OP(bit_reverse, BitReverse)
  // cl_khr_split_work_group_barrier
  _SPIRV_OP(intel_work_group_barrier_arrive, ControlBarrierArriveINTEL)
  _SPIRV_OP(intel_work_group_barrier_wait, ControlBarrierWaitINTEL)
#undef _SPIRV_OP
}

template <> void SPIRVMap<std::string, Op, OCL12Builtin>::init() {
#define _SPIRV_OP(x, y) add(#x, Op##y);
  _SPIRV_OP(add, AtomicIAdd)
  _SPIRV_OP(sub, AtomicISub)
  _SPIRV_OP(xchg, AtomicExchange)
  _SPIRV_OP(cmpxchg, AtomicCompareExchange)
  _SPIRV_OP(inc, AtomicIIncrement)
  _SPIRV_OP(dec, AtomicIDecrement)
  _SPIRV_OP(min, AtomicSMin)
  _SPIRV_OP(max, AtomicSMax)
  _SPIRV_OP(umin, AtomicUMin)
  _SPIRV_OP(umax, AtomicUMax)
  _SPIRV_OP(and, AtomicAnd)
  _SPIRV_OP(or, AtomicOr)
  _SPIRV_OP(xor, AtomicXor)
#undef _SPIRV_OP
}

// SPV_INTEL_device_side_avc_motion_estimation extension builtins
class SPIRVSubgroupsAVCIntelInst;
template <> void SPIRVMap<std::string, Op, SPIRVSubgroupsAVCIntelInst>::init() {
  // Here is a workaround for a bug in the specification:
  // 'avc' missed in 'intel_sub_group_avc' prefix.
  add("intel_sub_group_ime_ref_window_size",
      OpSubgroupAvcImeRefWindowSizeINTEL);

#define _SPIRV_OP(x, y) add("intel_sub_group_avc_" #x, OpSubgroupAvc##y##INTEL);
  // Initialization phase functions
  _SPIRV_OP(ime_initialize, ImeInitialize)
  _SPIRV_OP(fme_initialize, FmeInitialize)
  _SPIRV_OP(bme_initialize, BmeInitialize)
  _SPIRV_OP(sic_initialize, SicInitialize)

  // Result and payload types conversion functions
  _SPIRV_OP(mce_convert_to_ime_payload, MceConvertToImePayload)
  _SPIRV_OP(mce_convert_to_ime_result, MceConvertToImeResult)
  _SPIRV_OP(mce_convert_to_ref_payload, MceConvertToRefPayload)
  _SPIRV_OP(mce_convert_to_ref_result, MceConvertToRefResult)
  _SPIRV_OP(mce_convert_to_sic_payload, MceConvertToSicPayload)
  _SPIRV_OP(mce_convert_to_sic_result, MceConvertToSicResult)
  _SPIRV_OP(ime_convert_to_mce_payload, ImeConvertToMcePayload)
  _SPIRV_OP(ime_convert_to_mce_result, ImeConvertToMceResult)
  _SPIRV_OP(ref_convert_to_mce_payload, RefConvertToMcePayload)
  _SPIRV_OP(ref_convert_to_mce_result, RefConvertToMceResult)
  _SPIRV_OP(sic_convert_to_mce_payload, SicConvertToMcePayload)
  _SPIRV_OP(sic_convert_to_mce_result, SicConvertToMceResult)
#undef _SPIRV_OP

// MCE instructions
#define _SPIRV_OP(x, y)                                                        \
  add("intel_sub_group_avc_mce_" #x, OpSubgroupAvcMce##y##INTEL);
  _SPIRV_OP(get_default_inter_base_multi_reference_penalty,
            GetDefaultInterBaseMultiReferencePenalty)
  _SPIRV_OP(set_inter_base_multi_reference_penalty,
            SetInterBaseMultiReferencePenalty)
  _SPIRV_OP(get_default_inter_shape_penalty, GetDefaultInterShapePenalty)
  _SPIRV_OP(set_inter_shape_penalty, SetInterShapePenalty)
  _SPIRV_OP(get_default_inter_direction_penalty,
            GetDefaultInterDirectionPenalty)
  _SPIRV_OP(set_inter_direction_penalty, SetInterDirectionPenalty)
  _SPIRV_OP(get_default_intra_luma_shape_penalty,
            GetDefaultIntraLumaShapePenalty)
  _SPIRV_OP(get_default_inter_motion_vector_cost_table,
            GetDefaultInterMotionVectorCostTable)
  _SPIRV_OP(get_default_high_penalty_cost_table, GetDefaultHighPenaltyCostTable)
  _SPIRV_OP(get_default_medium_penalty_cost_table,
            GetDefaultMediumPenaltyCostTable)
  _SPIRV_OP(get_default_low_penalty_cost_table, GetDefaultLowPenaltyCostTable)
  _SPIRV_OP(set_motion_vector_cost_function, SetMotionVectorCostFunction)
  _SPIRV_OP(get_default_intra_luma_mode_penalty, GetDefaultIntraLumaModePenalty)
  _SPIRV_OP(get_default_non_dc_luma_intra_penalty,
            GetDefaultNonDcLumaIntraPenalty)
  _SPIRV_OP(get_default_intra_chroma_mode_base_penalty,
            GetDefaultIntraChromaModeBasePenalty)
  _SPIRV_OP(set_ac_only_haar, SetAcOnlyHaar)
  _SPIRV_OP(set_source_interlaced_field_polarity,
            SetSourceInterlacedFieldPolarity)
  _SPIRV_OP(set_single_reference_interlaced_field_polarity,
            SetSingleReferenceInterlacedFieldPolarity)
  _SPIRV_OP(set_dual_reference_interlaced_field_polarities,
            SetDualReferenceInterlacedFieldPolarities)
  _SPIRV_OP(get_motion_vectors, GetMotionVectors)
  _SPIRV_OP(get_inter_distortions, GetInterDistortions)
  _SPIRV_OP(get_best_inter_distortion, GetBestInterDistortions)
  _SPIRV_OP(get_inter_major_shape, GetInterMajorShape)
  _SPIRV_OP(get_inter_minor_shapes, GetInterMinorShape)
  _SPIRV_OP(get_inter_directions, GetInterDirections)
  _SPIRV_OP(get_inter_motion_vector_count, GetInterMotionVectorCount)
  _SPIRV_OP(get_inter_reference_ids, GetInterReferenceIds)
  _SPIRV_OP(get_inter_reference_interlaced_field_polarities,
            GetInterReferenceInterlacedFieldPolarities)
#undef _SPIRV_OP

// IME instructions
#define _SPIRV_OP(x, y)                                                        \
  add("intel_sub_group_avc_ime_" #x, OpSubgroupAvcIme##y##INTEL);
  _SPIRV_OP(set_single_reference, SetSingleReference)
  _SPIRV_OP(set_dual_reference, SetDualReference)
  _SPIRV_OP(ref_window_size, RefWindowSize)
  _SPIRV_OP(adjust_ref_offset, AdjustRefOffset)
  _SPIRV_OP(set_max_motion_vector_count, SetMaxMotionVectorCount)
  _SPIRV_OP(set_unidirectional_mix_disable, SetUnidirectionalMixDisable)
  _SPIRV_OP(set_early_search_termination_threshold,
            SetEarlySearchTerminationThreshold)
  _SPIRV_OP(set_weighted_sad, SetWeightedSad)
  _SPIRV_OP(evaluate_with_single_reference, EvaluateWithSingleReference)
  _SPIRV_OP(evaluate_with_dual_reference, EvaluateWithDualReference)
  _SPIRV_OP(evaluate_with_single_reference_streamin,
            EvaluateWithSingleReferenceStreamin)
  _SPIRV_OP(evaluate_with_dual_reference_streamin,
            EvaluateWithDualReferenceStreamin)
  _SPIRV_OP(evaluate_with_single_reference_streamout,
            EvaluateWithSingleReferenceStreamout)
  _SPIRV_OP(evaluate_with_dual_reference_streamout,
            EvaluateWithDualReferenceStreamout)
  _SPIRV_OP(evaluate_with_single_reference_streaminout,
            EvaluateWithSingleReferenceStreaminout)
  _SPIRV_OP(evaluate_with_dual_reference_streaminout,
            EvaluateWithDualReferenceStreaminout)
  _SPIRV_OP(get_single_reference_streamin, GetSingleReferenceStreamin)
  _SPIRV_OP(get_dual_reference_streamin, GetDualReferenceStreamin)
  _SPIRV_OP(strip_single_reference_streamout, StripSingleReferenceStreamout)
  _SPIRV_OP(strip_dual_reference_streamout, StripDualReferenceStreamout)
  _SPIRV_OP(get_border_reached, GetBorderReached)
  _SPIRV_OP(get_truncated_search_indication, GetTruncatedSearchIndication)
  _SPIRV_OP(get_unidirectional_early_search_termination,
            GetUnidirectionalEarlySearchTermination)
  _SPIRV_OP(get_weighting_pattern_minimum_motion_vector,
            GetWeightingPatternMinimumMotionVector)
  _SPIRV_OP(get_weighting_pattern_minimum_distortion,
            GetWeightingPatternMinimumDistortion)
#undef _SPIRV_OP

#define _SPIRV_OP(x, y)                                                        \
  add("intel_sub_group_avc_ime_get_streamout_major_shape_" #x,                 \
      OpSubgroupAvcImeGetStreamout##y##INTEL);
  _SPIRV_OP(motion_vectors_single_reference,
            SingleReferenceMajorShapeMotionVectors)
  _SPIRV_OP(distortions_single_reference, SingleReferenceMajorShapeDistortions)
  _SPIRV_OP(reference_ids_single_reference,
            SingleReferenceMajorShapeReferenceIds)
  _SPIRV_OP(motion_vectors_dual_reference, DualReferenceMajorShapeMotionVectors)
  _SPIRV_OP(distortions_dual_reference, DualReferenceMajorShapeDistortions)
  _SPIRV_OP(reference_ids_dual_reference, DualReferenceMajorShapeReferenceIds)
#undef _SPIRV_OP

// REF instructions
#define _SPIRV_OP(x, y)                                                        \
  add("intel_sub_group_avc_ref_" #x, OpSubgroupAvcRef##y##INTEL);
  _SPIRV_OP(set_bidirectional_mix_disable, SetBidirectionalMixDisable)
  _SPIRV_OP(set_bilinear_filter_enable, SetBilinearFilterEnable)
  _SPIRV_OP(evaluate_with_single_reference, EvaluateWithSingleReference)
  _SPIRV_OP(evaluate_with_dual_reference, EvaluateWithDualReference)
  _SPIRV_OP(evaluate_with_multi_reference, EvaluateWithMultiReference)
  _SPIRV_OP(evaluate_with_multi_reference_interlaced,
            EvaluateWithMultiReferenceInterlaced)
#undef _SPIRV_OP

// SIC instructions
#define _SPIRV_OP(x, y)                                                        \
  add("intel_sub_group_avc_sic_" #x, OpSubgroupAvcSic##y##INTEL);
  _SPIRV_OP(configure_skc, ConfigureSkc)
  _SPIRV_OP(configure_ipe_luma, ConfigureIpeLuma)
  _SPIRV_OP(configure_ipe_luma_chroma, ConfigureIpeLumaChroma)
  _SPIRV_OP(get_motion_vector_mask, GetMotionVectorMask)
  _SPIRV_OP(set_intra_luma_shape_penalty, SetIntraLumaShapePenalty)
  _SPIRV_OP(set_intra_luma_mode_cost_function, SetIntraLumaModeCostFunction)
  _SPIRV_OP(set_intra_chroma_mode_cost_function, SetIntraChromaModeCostFunction)
  _SPIRV_OP(set_skc_bilinear_filter_enable, SetBilinearFilterEnable)
  _SPIRV_OP(set_skc_forward_transform_enable, SetSkcForwardTransformEnable)
  _SPIRV_OP(set_block_based_raw_skip_sad, SetBlockBasedRawSkipSad)
  _SPIRV_OP(evaluate_ipe, EvaluateIpe)
  _SPIRV_OP(evaluate_with_single_reference, EvaluateWithSingleReference)
  _SPIRV_OP(evaluate_with_dual_reference, EvaluateWithDualReference)
  _SPIRV_OP(evaluate_with_multi_reference, EvaluateWithMultiReference)
  _SPIRV_OP(evaluate_with_multi_reference_interlaced,
            EvaluateWithMultiReferenceInterlaced)
  _SPIRV_OP(get_ipe_luma_shape, GetIpeLumaShape)
  _SPIRV_OP(get_best_ipe_luma_distortion, GetBestIpeLumaDistortion)
  _SPIRV_OP(get_best_ipe_chroma_distortion, GetBestIpeChromaDistortion)
  _SPIRV_OP(get_packed_ipe_luma_modes, GetPackedIpeLumaModes)
  _SPIRV_OP(get_ipe_chroma_mode, GetIpeChromaMode)
  _SPIRV_OP(get_packed_skc_luma_count_threshold, GetPackedSkcLumaCountThreshold)
  _SPIRV_OP(get_packed_skc_luma_sum_threshold, GetPackedSkcLumaSumThreshold)
  _SPIRV_OP(get_inter_raw_sads, GetInterRawSads)
#undef _SPIRV_OP
}

template <> void SPIRVMap<std::string, Op, OCLOpaqueType>::init() {
  add("opencl.event_t", OpTypeEvent);
  add("opencl.pipe_t", OpTypePipe);
  add("opencl.clk_event_t", OpTypeDeviceEvent);
  add("opencl.reserve_id_t", OpTypeReserveId);
  add("opencl.queue_t", OpTypeQueue);
  add("opencl.sampler_t", OpTypeSampler);
}

template <> void LLVMSPIRVAtomicRmwOpCodeMap::init() {
  add(llvm::AtomicRMWInst::Xchg, OpAtomicExchange);
  add(llvm::AtomicRMWInst::Add, OpAtomicIAdd);
  add(llvm::AtomicRMWInst::Sub, OpAtomicISub);
  add(llvm::AtomicRMWInst::And, OpAtomicAnd);
  add(llvm::AtomicRMWInst::Or, OpAtomicOr);
  add(llvm::AtomicRMWInst::Xor, OpAtomicXor);
  add(llvm::AtomicRMWInst::Max, OpAtomicSMax);
  add(llvm::AtomicRMWInst::Min, OpAtomicSMin);
  add(llvm::AtomicRMWInst::UMax, OpAtomicUMax);
  add(llvm::AtomicRMWInst::UMin, OpAtomicUMin);
  add(llvm::AtomicRMWInst::FAdd, OpAtomicFAddEXT);
  add(llvm::AtomicRMWInst::FMin, OpAtomicFMinEXT);
  add(llvm::AtomicRMWInst::FMax, OpAtomicFMaxEXT);
}

} // namespace SPIRV

///////////////////////////////////////////////////////////////////////////////
//
// Functions for getting builtin call info
//
///////////////////////////////////////////////////////////////////////////////

namespace OCLUtil {

AtomicWorkItemFenceLiterals getAtomicWorkItemFenceLiterals(CallInst *CI) {
  return std::make_tuple(getArgAsInt(CI, 0),
                         static_cast<OCLMemOrderKind>(getArgAsInt(CI, 1)),
                         static_cast<OCLScopeKind>(getArgAsInt(CI, 2)));
}

size_t getAtomicBuiltinNumMemoryOrderArgs(StringRef Name) {
  if (Name.starts_with("atomic_compare_exchange"))
    return 2;
  return 1;
}

size_t getSPIRVAtomicBuiltinNumMemoryOrderArgs(Op OC) {
  if (OC == OpAtomicCompareExchange || OC == OpAtomicCompareExchangeWeak)
    return 2;
  return 1;
}

// atomic_fetch_[add, sub, min, max] and atomic_fetch_[add, sub, min,
// max]_explicit functions declared in clang headers should be translated
// to corresponding FP-typed Atomic Instructions
bool isComputeAtomicOCLBuiltin(StringRef DemangledName) {
  if (!DemangledName.starts_with(kOCLBuiltinName::AtomicPrefix) &&
      !DemangledName.starts_with(kOCLBuiltinName::AtomPrefix))
    return false;

  return llvm::StringSwitch<bool>(DemangledName)
      .EndsWith("atomic_add", true)
      .EndsWith("atomic_sub", true)
      .EndsWith("atomic_min", true)
      .EndsWith("atomic_max", true)
      .EndsWith("atom_add", true)
      .EndsWith("atom_sub", true)
      .EndsWith("atom_min", true)
      .EndsWith("atom_max", true)
      .EndsWith("inc", true)
      .EndsWith("dec", true)
      .EndsWith("cmpxchg", true)
      .EndsWith("and", true)
      .EndsWith("or", true)
      .EndsWith("xor", true)
      .EndsWith("or_explicit", true)
      .EndsWith("xor_explicit", true)
      .EndsWith("and_explicit", true)
      .Default(false);
}

BarrierLiterals getBarrierLiterals(CallInst *CI) {
  auto N = CI->arg_size();
  assert(N == 1 || N == 2);

  StringRef DemangledName;
  assert(CI->getCalledFunction() && "Unexpected indirect call");
  if (!oclIsBuiltin(CI->getCalledFunction()->getName(), DemangledName)) {
    assert(0 &&
           "call must a builtin (work_group_barrier or sub_group_barrier)");
  }

  OCLScopeKind Scope = OCLMS_work_group;
  if (DemangledName == kOCLBuiltinName::SubGroupBarrier) {
    Scope = OCLMS_sub_group;
  }

  return std::make_tuple(getArgAsInt(CI, 0),
                         N == 1 ? OCLMS_work_group
                                : static_cast<OCLScopeKind>(getArgAsInt(CI, 1)),
                         Scope);
}

unsigned getExtOp(StringRef OrigName, StringRef GivenDemangledName) {
  std::string DemangledName{GivenDemangledName};
  if (DemangledName.empty() || !oclIsBuiltin(OrigName, GivenDemangledName))
    return ~0U;
  LLVM_DEBUG(dbgs() << "getExtOp: demangled name: " << DemangledName << '\n');
  OCLExtOpKind EOC;
  bool Found = OCLExtOpMap::rfind(DemangledName, &EOC);
  if (!Found) {
    std::string Prefix;
    switch (lastFuncParamType(OrigName)) {
    case ParamType::UNSIGNED:
      Prefix = "u_";
      break;
    case ParamType::SIGNED:
      Prefix = "s_";
      break;
    case ParamType::FLOAT:
      Prefix = "f";
      break;
    case ParamType::UNKNOWN:
      break;
    }
    Found = OCLExtOpMap::rfind(Prefix + DemangledName, &EOC);
  }
  if (Found)
    return EOC;
  else
    return ~0U;
}

///////////////////////////////////////////////////////////////////////////////
//
// Functions for getting module info
//
///////////////////////////////////////////////////////////////////////////////

unsigned encodeOCLVer(unsigned short Major, unsigned char Minor,
                      unsigned char Rev) {
  return (Major * 100 + Minor) * 1000 + Rev;
}

std::tuple<unsigned short, unsigned char, unsigned char>
decodeOCLVer(unsigned Ver) {
  unsigned short Major = Ver / 100000;
  unsigned char Minor = (Ver % 100000) / 1000;
  unsigned char Rev = Ver % 1000;
  return std::make_tuple(Major, Minor, Rev);
}

unsigned getOCLVersion(Module *M, bool AllowMulti) {
  NamedMDNode *NamedMDOCLVer = M->getNamedMetadata(kSPIR2MD::OCLVer);
  NamedMDNode *NamedMDOCLCXXVer = M->getNamedMetadata(kSPIR2MD::OCLCXXVer);
  if (!NamedMDOCLVer && !NamedMDOCLCXXVer)
    return 0;

  // If the module was linked with another module, there may be multiple
  // operands.
  auto GetVerPair = [](unsigned I, NamedMDNode *NamedMD) {
    auto *MD = NamedMD->getOperand(I);
    return std::make_pair(getMDOperandAsInt(MD, 0), getMDOperandAsInt(MD, 1));
  };
  auto GetVer = [=](NamedMDNode *NamedMD) {
    assert(NamedMD->getNumOperands() && "Invalid SPIR");
    if (!AllowMulti && NamedMD->getNumOperands() != 1)
      report_fatal_error(
          llvm::Twine("Multiple OCL version metadata not allowed"));
    auto Ver = GetVerPair(0, NamedMD);
    for (unsigned I = 1, E = NamedMD->getNumOperands(); I != E; ++I)
      if (Ver != GetVerPair(I, NamedMD))
        report_fatal_error(llvm::Twine("OCL version mismatch"));
    return encodeOCLVer(Ver.first, Ver.second, 0);
  };
  unsigned OCLVer = NamedMDOCLVer ? GetVer(NamedMDOCLVer) : 0;
  unsigned OCLCXXVer = NamedMDOCLCXXVer ? GetVer(NamedMDOCLCXXVer) : 0;
  // Check if OCLCXXVer is compatible with OCLVer
  if (OCLVer && OCLCXXVer) {
    if ((OCLVer == kOCLVer::CL20 && OCLCXXVer == kOCLVer::CLCXX10) ||
        (OCLVer == kOCLVer::CL30 && OCLCXXVer == kOCLVer::CLCXX2021))
      return OCLCXXVer;
    report_fatal_error(llvm::Twine(
        "opencl cxx version is not compatible with opencl c version!"));
  }
  return OCLVer;
}

SmallVector<unsigned, 3> decodeMDNode(MDNode *N) {
  if (N == NULL)
    return {};
  size_t NumOperands = N->getNumOperands();
  SmallVector<unsigned, 3> ReadVals;
  ReadVals.reserve(NumOperands);
  for (unsigned I = 0; I < NumOperands; ++I)
    ReadVals.push_back(getMDOperandAsInt(N, I));
  return ReadVals;
}

/// Encode LLVM type by SPIR-V execution mode VecTypeHint
unsigned encodeVecTypeHint(Type *Ty) {
  if (Ty->isHalfTy())
    return 4;
  if (Ty->isFloatTy())
    return 5;
  if (Ty->isDoubleTy())
    return 6;
  if (IntegerType *IntTy = dyn_cast<IntegerType>(Ty)) {
    switch (IntTy->getIntegerBitWidth()) {
    case 8:
      return 0;
    case 16:
      return 1;
    case 32:
      return 2;
    case 64:
      return 3;
    default:
      llvm_unreachable("invalid integer type");
    }
  }
  if (FixedVectorType *VecTy = dyn_cast<FixedVectorType>(Ty)) {
    Type *EleTy = VecTy->getElementType();
    unsigned Size = VecTy->getNumElements();
    return Size << 16 | encodeVecTypeHint(EleTy);
  }
  llvm_unreachable("invalid type");
  return ~0U;
}

Type *decodeVecTypeHint(LLVMContext &C, unsigned Code) {
  unsigned VecWidth = Code >> 16;
  unsigned Scalar = Code & 0xFFFF;
  Type *ST = nullptr;
  switch (Scalar) {
  case 0:
  case 1:
  case 2:
  case 3:
    ST = IntegerType::get(C, 1 << (3 + Scalar));
    break;
  case 4:
    ST = Type::getHalfTy(C);
    break;
  case 5:
    ST = Type::getFloatTy(C);
    break;
  case 6:
    ST = Type::getDoubleTy(C);
    break;
  default:
    llvm_unreachable("Invalid vec type hint");
    return nullptr;
  }
  if (VecWidth < 1)
    return ST;
  return FixedVectorType::get(ST, VecWidth);
}

unsigned transVecTypeHint(MDNode *Node) {
  return encodeVecTypeHint(getMDOperandAsType(Node, 0));
}

SPIRAddressSpace getOCLOpaqueTypeAddrSpace(Op OpCode) {
  switch ((unsigned)OpCode) {
  case OpTypeQueue:
    return SPIRV_QUEUE_T_ADDR_SPACE;
  case OpTypeEvent:
    return SPIRV_EVENT_T_ADDR_SPACE;
  case OpTypeDeviceEvent:
    return SPIRV_CLK_EVENT_T_ADDR_SPACE;
  case OpTypeReserveId:
    return SPIRV_RESERVE_ID_T_ADDR_SPACE;
  case OpTypePipe:
  case OpTypePipeStorage:
    return SPIRV_PIPE_ADDR_SPACE;
  case OpTypeImage:
  case OpTypeSampledImage:
  case OpTypeVmeImageINTEL:
    return SPIRV_IMAGE_ADDR_SPACE;
  case OpConstantSampler:
  case OpTypeSampler:
    return SPIRV_SAMPLER_T_ADDR_SPACE;
  case OpTypeCooperativeMatrixKHR:
  case internal::OpTypeTaskSequenceINTEL:
    return SPIRAS_Global;
  default:
    if (isSubgroupAvcINTELTypeOpCode(OpCode))
      return SPIRV_AVC_INTEL_T_ADDR_SPACE;
    assert(false && "No address space is determined for some OCL type");
    return SPIRV_OCL_SPECIAL_TYPES_DEFAULT_ADDR_SPACE;
  }
}

static SPIR::TypeAttributeEnum mapAddrSpaceEnums(SPIRAddressSpace Addrspace) {
  switch (Addrspace) {
  case SPIRAS_Private:
    return SPIR::ATTR_PRIVATE;
  case SPIRAS_Global:
    return SPIR::ATTR_GLOBAL;
  case SPIRAS_Constant:
    return SPIR::ATTR_CONSTANT;
  case SPIRAS_Local:
    return SPIR::ATTR_LOCAL;
  case SPIRAS_Generic:
    return SPIR::ATTR_GENERIC;
  case SPIRAS_GlobalDevice:
    return SPIR::ATTR_GLOBAL_DEVICE;
  case SPIRAS_GlobalHost:
    return SPIR::ATTR_GLOBAL_HOST;
  default:
    llvm_unreachable("Invalid addrspace enum member");
  }
  return SPIR::ATTR_NONE;
}

SPIR::TypeAttributeEnum
getOCLOpaqueTypeAddrSpace(SPIR::TypePrimitiveEnum Prim) {
  switch (Prim) {
  case SPIR::PRIMITIVE_QUEUE_T:
    return mapAddrSpaceEnums(SPIRV_QUEUE_T_ADDR_SPACE);
  case SPIR::PRIMITIVE_EVENT_T:
    return mapAddrSpaceEnums(SPIRV_EVENT_T_ADDR_SPACE);
  case SPIR::PRIMITIVE_CLK_EVENT_T:
    return mapAddrSpaceEnums(SPIRV_CLK_EVENT_T_ADDR_SPACE);
  case SPIR::PRIMITIVE_RESERVE_ID_T:
    return mapAddrSpaceEnums(SPIRV_RESERVE_ID_T_ADDR_SPACE);
  case SPIR::PRIMITIVE_PIPE_RO_T:
  case SPIR::PRIMITIVE_PIPE_WO_T:
    return mapAddrSpaceEnums(SPIRV_PIPE_ADDR_SPACE);
  case SPIR::PRIMITIVE_IMAGE1D_RO_T:
  case SPIR::PRIMITIVE_IMAGE1D_ARRAY_RO_T:
  case SPIR::PRIMITIVE_IMAGE1D_BUFFER_RO_T:
  case SPIR::PRIMITIVE_IMAGE2D_RO_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_RO_T:
  case SPIR::PRIMITIVE_IMAGE2D_DEPTH_RO_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RO_T:
  case SPIR::PRIMITIVE_IMAGE2D_MSAA_RO_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_RO_T:
  case SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_RO_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RO_T:
  case SPIR::PRIMITIVE_IMAGE3D_RO_T:
  case SPIR::PRIMITIVE_IMAGE1D_WO_T:
  case SPIR::PRIMITIVE_IMAGE1D_ARRAY_WO_T:
  case SPIR::PRIMITIVE_IMAGE1D_BUFFER_WO_T:
  case SPIR::PRIMITIVE_IMAGE2D_WO_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_WO_T:
  case SPIR::PRIMITIVE_IMAGE2D_DEPTH_WO_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_WO_T:
  case SPIR::PRIMITIVE_IMAGE2D_MSAA_WO_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_WO_T:
  case SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_WO_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_WO_T:
  case SPIR::PRIMITIVE_IMAGE3D_WO_T:
  case SPIR::PRIMITIVE_IMAGE1D_RW_T:
  case SPIR::PRIMITIVE_IMAGE1D_ARRAY_RW_T:
  case SPIR::PRIMITIVE_IMAGE1D_BUFFER_RW_T:
  case SPIR::PRIMITIVE_IMAGE2D_RW_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_RW_T:
  case SPIR::PRIMITIVE_IMAGE2D_DEPTH_RW_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_DEPTH_RW_T:
  case SPIR::PRIMITIVE_IMAGE2D_MSAA_RW_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_RW_T:
  case SPIR::PRIMITIVE_IMAGE2D_MSAA_DEPTH_RW_T:
  case SPIR::PRIMITIVE_IMAGE2D_ARRAY_MSAA_DEPTH_RW_T:
  case SPIR::PRIMITIVE_IMAGE3D_RW_T:
    return mapAddrSpaceEnums(SPIRV_IMAGE_ADDR_SPACE);
  default:
    llvm_unreachable("No address space is determined for a SPIR primitive");
  }
  return SPIR::ATTR_NONE;
}

// Fetch type of invoke function passed to device execution built-ins
static FunctionType *getBlockInvokeTy(Function *F, unsigned BlockIdx) {
  auto Params = F->getFunctionType()->params();
  PointerType *FuncPtr = cast<PointerType>(Params[BlockIdx]);
  return FunctionType::get(FuncPtr, Params, false);
}

void OCLBuiltinFuncMangleInfo::init(StringRef UniqName) {
  // Make a local copy as we will modify the string in init function
  std::string TempStorage = UniqName.str();
  auto NameRef = StringRef(TempStorage);

  // Helper functions to erase substrings from NameRef (i.e. TempStorage)
  auto EraseSubstring = [&NameRef, &TempStorage](const std::string &ToErase) {
    size_t Pos = TempStorage.find(ToErase);
    if (Pos != std::string::npos) {
      TempStorage.erase(Pos, ToErase.length());
      // re-take StringRef as TempStorage was updated
      NameRef = StringRef(TempStorage);
    }
  };
  auto EraseSymbol = [&NameRef, &TempStorage](size_t Index) {
    TempStorage.erase(Index, 1);
    // re-take StringRef as TempStorage was updated
    NameRef = StringRef(TempStorage);
  };

  if (NameRef.starts_with("async_work_group")) {
    addUnsignedArg(-1);
    setArgAttr(1, SPIR::ATTR_CONST);
  } else if (NameRef.starts_with("printf"))
    setVarArg(1);
  else if (NameRef.starts_with("write_imageui"))
    addUnsignedArg(2);
  else if (NameRef == "prefetch") {
    addUnsignedArg(1);
    setArgAttr(0, SPIR::ATTR_CONST);
  } else if (NameRef == "get_kernel_work_group_size" ||
             NameRef == "get_kernel_preferred_work_group_size_multiple") {
    assert(F && "lack of necessary information");
    const size_t BlockArgIdx = 0;
    FunctionType *InvokeTy = getBlockInvokeTy(F, BlockArgIdx);
    if (InvokeTy->getNumParams() > 1)
      setLocalArgBlock(BlockArgIdx);
  } else if (NameRef.starts_with("__enqueue_kernel")) {
    // clang doesn't mangle enqueue_kernel builtins
    setAsDontMangle();
  } else if (NameRef.starts_with("get_") || NameRef == "nan" ||
             NameRef == "mem_fence" || NameRef.starts_with("shuffle")) {
    addUnsignedArg(-1);
    if (NameRef.starts_with(kOCLBuiltinName::GetFence)) {
      setArgAttr(0, SPIR::ATTR_CONST);
      addVoidPtrArg(0);
    }
  } else if (NameRef.contains("barrier")) {
    addUnsignedArg(0);
    if (NameRef == "work_group_barrier" || NameRef == "sub_group_barrier" ||
        NameRef == "intel_work_group_barrier_arrive" ||
        NameRef == "intel_work_group_barrier_wait")
      setEnumArg(1, SPIR::PRIMITIVE_MEMORY_SCOPE);
  } else if (NameRef.starts_with("atomic_work_item_fence")) {
    addUnsignedArg(0);
    setEnumArg(1, SPIR::PRIMITIVE_MEMORY_ORDER);
    setEnumArg(2, SPIR::PRIMITIVE_MEMORY_SCOPE);
  } else if (NameRef.starts_with("atom_")) {
    setArgAttr(0, SPIR::ATTR_VOLATILE);
    if (NameRef.ends_with("_umax") || NameRef.ends_with("_umin")) {
      addUnsignedArg(-1);
      // We need to remove u to match OpenCL C built-in function name
      EraseSymbol(5);
    }
  } else if (NameRef.starts_with("atomic")) {
    setArgAttr(0, SPIR::ATTR_VOLATILE);
    if (NameRef.contains("_umax") || NameRef.contains("_umin")) {
      addUnsignedArg(-1);
      // We need to remove u to match OpenCL C built-in function name
      if (NameRef.contains("_fetch"))
        EraseSymbol(13);
      else
        EraseSymbol(7);
    }
    if (NameRef.contains("store_explicit") ||
        NameRef.contains("exchange_explicit") ||
        (NameRef.starts_with("atomic_fetch") && NameRef.contains("explicit"))) {
      setEnumArg(2, SPIR::PRIMITIVE_MEMORY_ORDER);
      setEnumArg(3, SPIR::PRIMITIVE_MEMORY_SCOPE);
    } else if (NameRef.contains("load_explicit") ||
               (NameRef.starts_with("atomic_flag") &&
                NameRef.contains("explicit"))) {
      setEnumArg(1, SPIR::PRIMITIVE_MEMORY_ORDER);
      setEnumArg(2, SPIR::PRIMITIVE_MEMORY_SCOPE);
    } else if (NameRef.ends_with("compare_exchange_strong_explicit") ||
               NameRef.ends_with("compare_exchange_weak_explicit")) {
      setEnumArg(3, SPIR::PRIMITIVE_MEMORY_ORDER);
      setEnumArg(4, SPIR::PRIMITIVE_MEMORY_ORDER);
      setEnumArg(5, SPIR::PRIMITIVE_MEMORY_SCOPE);
    }
    // Don't set atomic property to the first argument of 1.2 atomic
    // built-ins.
    if (!NameRef.ends_with("xchg") && // covers _cmpxchg too
        (NameRef.contains("fetch") ||
         !(NameRef.ends_with("_add") || NameRef.ends_with("_sub") ||
           NameRef.ends_with("_inc") || NameRef.ends_with("_dec") ||
           NameRef.ends_with("_min") || NameRef.ends_with("_max") ||
           NameRef.ends_with("_and") || NameRef.ends_with("_or") ||
           NameRef.ends_with("_xor")))) {
      addAtomicArg(0);
    }
  } else if (NameRef.starts_with("uconvert_")) {
    addUnsignedArg(0);
    NameRef = NameRef.drop_front(1);
    UnmangledName.erase(0, 1);
  } else if (NameRef.starts_with("s_")) {
    if (NameRef == "s_upsample")
      addUnsignedArg(1);
    NameRef = NameRef.drop_front(2);
  } else if (NameRef.starts_with("u_")) {
    addUnsignedArg(-1);
    NameRef = NameRef.drop_front(2);
  } else if (NameRef == "fclamp") {
    NameRef = NameRef.drop_front(1);
  }
  // handle [read|write]pipe builtins (plus two i32 literal args
  // required by SPIR 2.0 provisional specification):
  else if (NameRef == "read_pipe_2" || NameRef == "write_pipe_2") {
    // with 2 arguments (plus two i32 literals):
    // int read_pipe (read_only pipe gentype p, gentype *ptr)
    // int write_pipe (write_only pipe gentype p, const gentype *ptr)
    addVoidPtrArg(1);
    addUnsignedArg(2);
    addUnsignedArg(3);
    // OpenCL-like representation of blocking pipes
  } else if (NameRef == "read_pipe_2_bl" || NameRef == "write_pipe_2_bl") {
    // with 2 arguments (plus two i32 literals):
    // int read_pipe_bl (read_only pipe gentype p, gentype *ptr)
    // int write_pipe_bl (write_only pipe gentype p, const gentype *ptr)
    addVoidPtrArg(1);
    addUnsignedArg(2);
    addUnsignedArg(3);
  } else if (NameRef == "read_pipe_4" || NameRef == "write_pipe_4") {
    // with 4 arguments (plus two i32 literals):
    // int read_pipe (read_only pipe gentype p, reserve_id_t reserve_id, uint
    // index, gentype *ptr) int write_pipe (write_only pipe gentype p,
    // reserve_id_t reserve_id, uint index, const gentype *ptr)
    addUnsignedArg(2);
    addVoidPtrArg(3);
    addUnsignedArg(4);
    addUnsignedArg(5);
  } else if (NameRef.contains("reserve_read_pipe") ||
             NameRef.contains("reserve_write_pipe")) {
    // process [|work_group|sub_group]reserve[read|write]pipe builtins
    addUnsignedArg(1);
    addUnsignedArg(2);
    addUnsignedArg(3);
  } else if (NameRef.contains("commit_read_pipe") ||
             NameRef.contains("commit_write_pipe")) {
    // process [|work_group|sub_group]commit[read|write]pipe builtins
    addUnsignedArg(2);
    addUnsignedArg(3);
  } else if (NameRef == "capture_event_profiling_info") {
    addVoidPtrArg(2);
    setEnumArg(1, SPIR::PRIMITIVE_CLK_PROFILING_INFO);
  } else if (NameRef == "enqueue_marker") {
    setArgAttr(2, SPIR::ATTR_CONST);
    addUnsignedArg(1);
  } else if (NameRef.starts_with("vload")) {
    addUnsignedArg(0);
    setArgAttr(1, SPIR::ATTR_CONST);
  } else if (NameRef.starts_with("vstore")) {
    addUnsignedArg(1);
  } else if (NameRef.starts_with("ndrange_")) {
    addUnsignedArgs(0, 2);
    if (NameRef[8] == '2' || NameRef[8] == '3') {
      setArgAttr(0, SPIR::ATTR_CONST);
      setArgAttr(1, SPIR::ATTR_CONST);
      setArgAttr(2, SPIR::ATTR_CONST);
    }
  } else if (NameRef.contains("umax")) {
    addUnsignedArg(-1);
    EraseSymbol(NameRef.find("umax"));
  } else if (NameRef.contains("umin")) {
    addUnsignedArg(-1);
    EraseSymbol(NameRef.find("umin"));
  } else if (NameRef.contains("broadcast")) {
    addUnsignedArg(-1);
  } else if (NameRef.starts_with(kOCLBuiltinName::SampledReadImage)) {
    if (!NameRef.consume_front(kOCLBuiltinName::Sampled))
      report_fatal_error(llvm::Twine("Builtin name illformed"));
    addSamplerArg(1);
  } else if (NameRef.contains(kOCLSubgroupsAVCIntel::Prefix)) {
    if (NameRef.contains("evaluate_ipe"))
      addSamplerArg(1);
    else if (NameRef.contains("evaluate_with_single_reference"))
      addSamplerArg(2);
    else if (NameRef.contains("evaluate_with_multi_reference")) {
      addUnsignedArg(1);
      std::string PostFix = "_interlaced";
      if (NameRef.contains(PostFix)) {
        addUnsignedArg(2);
        addSamplerArg(3);
        EraseSubstring(PostFix);
      } else
        addSamplerArg(2);
    } else if (NameRef.contains("evaluate_with_dual_reference"))
      addSamplerArg(3);
    else if (NameRef.contains("fme_initialize"))
      addUnsignedArgs(0, 6);
    else if (NameRef.contains("bme_initialize"))
      addUnsignedArgs(0, 7);
    else if (NameRef.contains("set_inter_base_multi_reference_penalty") ||
             NameRef.contains("set_inter_shape_penalty") ||
             NameRef.contains("set_inter_direction_penalty"))
      addUnsignedArg(0);
    else if (NameRef.contains("set_motion_vector_cost_function"))
      addUnsignedArgs(0, 2);
    else if (NameRef.contains("interlaced_field_polarity"))
      addUnsignedArg(0);
    else if (NameRef.contains("interlaced_field_polarities"))
      addUnsignedArgs(0, 1);
    else if (NameRef.contains(kOCLSubgroupsAVCIntel::MCEPrefix)) {
      if (NameRef.contains("get_default"))
        addUnsignedArgs(0, 1);
    } else if (NameRef.contains(kOCLSubgroupsAVCIntel::IMEPrefix)) {
      if (NameRef.contains("initialize"))
        addUnsignedArgs(0, 2);
      else if (NameRef.contains("set_single_reference"))
        addUnsignedArg(1);
      else if (NameRef.contains("set_dual_reference"))
        addUnsignedArg(2);
      else if (NameRef.contains("set_weighted_sad") ||
               NameRef.contains("set_early_search_termination_threshold"))
        addUnsignedArg(0);
      else if (NameRef.contains("adjust_ref_offset"))
        addUnsignedArgs(1, 3);
      else if (NameRef.contains("set_max_motion_vector_count") ||
               NameRef.contains("get_border_reached"))
        addUnsignedArg(0);
      else if (NameRef.contains("shape_distortions") ||
               NameRef.contains("shape_motion_vectors") ||
               NameRef.contains("shape_reference_ids")) {
        if (NameRef.contains("single_reference")) {
          addUnsignedArg(1);
          EraseSubstring("_single_reference");
        } else if (NameRef.contains("dual_reference")) {
          addUnsignedArgs(1, 2);
          EraseSubstring("_dual_reference");
        }
      } else if (NameRef.contains("ref_window_size"))
        addUnsignedArg(0);
    } else if (NameRef.contains(kOCLSubgroupsAVCIntel::SICPrefix)) {
      if (NameRef.contains("initialize") ||
          NameRef.contains("set_intra_luma_shape_penalty"))
        addUnsignedArg(0);
      else if (NameRef.contains("configure_ipe")) {
        if (NameRef.contains("_luma")) {
          addUnsignedArgs(0, 6);
          EraseSubstring("_luma");
        }
        if (NameRef.contains("_chroma")) {
          addUnsignedArgs(7, 9);
          EraseSubstring("_chroma");
        }
      } else if (NameRef.contains("configure_skc"))
        addUnsignedArgs(0, 4);
      else if (NameRef.contains("set_skc")) {
        if (NameRef.contains("forward_transform_enable"))
          addUnsignedArg(0);
      } else if (NameRef.contains("set_block")) {
        if (NameRef.contains("based_raw_skip_sad"))
          addUnsignedArg(0);
      } else if (NameRef.contains("get_motion_vector_mask")) {
        addUnsignedArgs(0, 1);
      } else if (NameRef.contains("luma_mode_cost_function"))
        addUnsignedArgs(0, 2);
      else if (NameRef.contains("chroma_mode_cost_function"))
        addUnsignedArg(0);
    }
  } else if (NameRef.starts_with("intel_sub_group_shuffle")) {
    if (NameRef.ends_with("_down") || NameRef.ends_with("_up"))
      addUnsignedArg(2);
    else
      addUnsignedArg(1);
  } else if (NameRef.starts_with("intel_sub_group_block_write")) {
    // distinguish write to image and other data types based on number of
    // arguments--images have one more argument.
    if (F->getFunctionType()->getNumParams() == 2) {
      addUnsignedArg(0);
      addUnsignedArg(1);
    } else {
      addUnsignedArg(2);
    }
  } else if (NameRef.starts_with("intel_sub_group_block_read")) {
    // distinguish read from image and other data types based on number of
    // arguments--images have one more argument.
    if (F->getFunctionType()->getNumParams() == 1) {
      setArgAttr(0, SPIR::ATTR_CONST);
      addUnsignedArg(0);
    }
  } else if (NameRef.starts_with("intel_sub_group_media_block_write")) {
    addUnsignedArg(3);
  } else if (NameRef.starts_with(kOCLBuiltinName::SubGroupPrefix)) {
    if (NameRef.contains("ballot")) {
      if (NameRef.contains("inverse") || NameRef.contains("bit_count") ||
          NameRef.contains("inclusive_scan") ||
          NameRef.contains("exclusive_scan") || NameRef.contains("find_lsb") ||
          NameRef.contains("find_msb"))
        addUnsignedArg(0);
      else if (NameRef.contains("bit_extract")) {
        addUnsignedArgs(0, 1);
      }
    } else if (NameRef.starts_with("sub_group_clustered_rotate")) {
      addUnsignedArg(2);
    } else if (NameRef.contains("shuffle") || NameRef.contains("clustered"))
      addUnsignedArg(1);
  } else if (NameRef.starts_with("bitfield_insert")) {
    addUnsignedArgs(2, 3);
  } else if (NameRef.starts_with("bitfield_extract_signed") ||
             NameRef.starts_with("bitfield_extract_unsigned")) {
    addUnsignedArgs(1, 2);
  } else if (NameRef.starts_with("dot_")) {
    if (NameRef.contains("4x8packed")) {
      addUnsignedArgs(0, 1);
      if (NameRef == "dot_acc_sat_4x8packed_uu_uint")
        addUnsignedArg(2);
    } else {
      if (NameRef.ends_with("_uu")) {
        addUnsignedArgs(0, 1);
        if (NameRef.starts_with("dot_acc_sat"))
          addUnsignedArg(2);
      } else if (NameRef.ends_with("_su"))
        addUnsignedArg(1);
      NameRef = NameRef.drop_back(std::string("_uu").length());
    }
  }

  // Store the final version of a function name
  UnmangledName = NameRef.str();
}

std::unique_ptr<OCLBuiltinFuncMangleInfo> makeMangler(Function &F) {
  return std::make_unique<OCLBuiltinFuncMangleInfo>(&F);
}

bool isSamplerTy(Type *Ty) {
  if (auto *TPT = dyn_cast_or_null<TypedPointerType>(Ty)) {
    auto *STy = dyn_cast_or_null<StructType>(TPT->getElementType());
    return STy && STy->hasName() && STy->getName() == kSPR2TypeName::Sampler;
  }
  if (auto *TET = dyn_cast_or_null<TargetExtType>(Ty)) {
    return TET->getName() == "spirv.Sampler";
  }
  return false;
}

bool isPipeOrAddressSpaceCastBI(const StringRef MangledName) {
  return MangledName == "write_pipe_2" || MangledName == "read_pipe_2" ||
         MangledName == "write_pipe_2_bl" || MangledName == "read_pipe_2_bl" ||
         MangledName == "write_pipe_4" || MangledName == "read_pipe_4" ||
         MangledName == "reserve_write_pipe" ||
         MangledName == "reserve_read_pipe" ||
         MangledName == "commit_write_pipe" ||
         MangledName == "commit_read_pipe" ||
         MangledName == "work_group_reserve_write_pipe" ||
         MangledName == "work_group_reserve_read_pipe" ||
         MangledName == "work_group_commit_write_pipe" ||
         MangledName == "work_group_commit_read_pipe" ||
         MangledName == "get_pipe_num_packets_ro" ||
         MangledName == "get_pipe_max_packets_ro" ||
         MangledName == "get_pipe_num_packets_wo" ||
         MangledName == "get_pipe_max_packets_wo" ||
         MangledName == "sub_group_reserve_write_pipe" ||
         MangledName == "sub_group_reserve_read_pipe" ||
         MangledName == "sub_group_commit_write_pipe" ||
         MangledName == "sub_group_commit_read_pipe" ||
         MangledName == "to_global" || MangledName == "to_local" ||
         MangledName == "to_private";
}

bool isEnqueueKernelBI(const StringRef MangledName) {
  return MangledName == "__enqueue_kernel_basic" ||
         MangledName == "__enqueue_kernel_basic_events" ||
         MangledName == "__enqueue_kernel_varargs" ||
         MangledName == "__enqueue_kernel_events_varargs";
}

bool isKernelQueryBI(const StringRef MangledName) {
  return MangledName == "__get_kernel_work_group_size_impl" ||
         MangledName == "__get_kernel_sub_group_count_for_ndrange_impl" ||
         MangledName == "__get_kernel_max_sub_group_size_for_ndrange_impl" ||
         MangledName == "__get_kernel_preferred_work_group_size_multiple_impl";
}

// isUnfusedMulAdd checks if we have the following (most common for fp
// contranction) pattern in LLVM IR:
//
//   %mul = fmul float %a, %b
//   %add = fadd float %mul, %c
//
// This pattern indicates that fp contraction could have been disabled by
// #pragma OPENCL FP_CONTRACT OFF. When contraction is enabled (by a pragma or
// by clang's -ffp-contract=fast), clang would generate:
//
//   %0 = call float @llvm.fmuladd.f32(float %a, float %b, float %c)
//
// or
//
//   %mul = fmul contract float %a, %b
//   %add = fadd contract float %mul, %c
//
// Note that optimizations may form an unfused fmuladd from fadd+load or
// fadd+call, so this check is quite restrictive (see the comment below).
//
bool isUnfusedMulAdd(BinaryOperator *B) {
  if (B->getOpcode() != Instruction::FAdd &&
      B->getOpcode() != Instruction::FSub)
    return false;

  if (B->hasAllowContract()) {
    // If this fadd or fsub itself has a contract flag, the operation can be
    // contracted regardless of the operands.
    return false;
  }

  // Otherwise, we cannot easily tell if the operation can be a candidate for
  // contraction or not. Consider the following cases:
  //
  //   %mul = alloca float
  //   %t1 = fmul float %a, %b
  //   store float* %mul, float %t
  //   %t2 = load %mul
  //   %r = fadd float %t2, %c
  //
  // LLVM IR does not allow %r to be contracted. However, after an optimization
  // it becomes a candidate for contraction if ContractionOFF is not set in
  // SPIR-V:
  //
  //   %t1 = fmul float %a, %b
  //   %r = fadd float %t1, %c
  //
  // To be on a safe side, we disallow everything that is even remotely similar
  // to fmul + fadd.
  return true;
}

std::string getIntelSubgroupBlockDataPostfix(unsigned ElementBitSize,
                                             unsigned VectorNumElements) {
  std::ostringstream OSS;
  switch (ElementBitSize) {
  case 8:
    OSS << "_uc";
    break;
  case 16:
    OSS << "_us";
    break;
  case 32:
    // Intentionally does nothing since _ui variant is only an alias.
    break;
  case 64:
    OSS << "_ul";
    break;
  default:
    llvm_unreachable(
        "Incorrect data bitsize for intel_subgroup_block builtins");
  }
  switch (VectorNumElements) {
  case 1:
    break;
  case 2:
  case 4:
  case 8:
    OSS << VectorNumElements;
    break;
  case 16:
    assert((ElementBitSize == 8 || ElementBitSize == 16) &&
           "16 elements vector allowed only for char and short builtins");
    OSS << VectorNumElements;
    break;
  default:
    llvm_unreachable(
        "Incorrect vector length for intel_subgroup_block builtins");
  }
  return OSS.str();
}

void insertImageNameAccessQualifier(SPIRVAccessQualifierKind Acc,
                                    std::string &Name) {
  std::string QName = rmap<std::string>(Acc);
  // transform: read_only -> ro, write_only -> wo, read_write -> rw
  QName = QName.substr(0, 1) + QName.substr(QName.find("_") + 1, 1) + "_";
  assert(!Name.empty() && "image name should not be empty");
  Name.insert(Name.size() - 1, QName);
}
} // namespace OCLUtil

Value *SPIRV::transOCLMemScopeIntoSPIRVScope(Value *MemScope,
                                             std::optional<int> DefaultCase,
                                             Instruction *InsertBefore) {
  if (auto *C = dyn_cast<ConstantInt>(MemScope)) {
    return ConstantInt::get(
        C->getType(), map<Scope>(static_cast<OCLScopeKind>(C->getZExtValue())));
  }

  // If memory_scope is not a constant, then we have to insert dynamic mapping:
  return getOrCreateSwitchFunc(kSPIRVName::TranslateOCLMemScope, MemScope,
                               OCLMemScopeMap::getMap(), /* IsReverse */ false,
                               DefaultCase, InsertBefore);
}

Value *
SPIRV::transOCLMemOrderIntoSPIRVMemorySemantics(Value *MemOrder,
                                                std::optional<int> DefaultCase,
                                                Instruction *InsertBefore) {
  if (auto *C = dyn_cast<ConstantInt>(MemOrder)) {
    return ConstantInt::get(
        C->getType(), mapOCLMemSemanticToSPIRV(
                          0, static_cast<OCLMemOrderKind>(C->getZExtValue())));
  }

  return getOrCreateSwitchFunc(kSPIRVName::TranslateOCLMemOrder, MemOrder,
                               OCLMemOrderMap::getMap(), /* IsReverse */ false,
                               DefaultCase, InsertBefore);
}

Value *
SPIRV::transSPIRVMemoryScopeIntoOCLMemoryScope(Value *MemScope,
                                               Instruction *InsertBefore) {
  if (auto *C = dyn_cast<ConstantInt>(MemScope)) {
    return ConstantInt::get(C->getType(), rmap<OCLScopeKind>(static_cast<Scope>(
                                              C->getZExtValue())));
  }

  if (auto *CI = dyn_cast<CallInst>(MemScope)) {
    Function *F = CI->getCalledFunction();
    if (F && F->getName() == kSPIRVName::TranslateOCLMemScope) {
      // In case the SPIR-V module was created from an OpenCL program by
      // *this* SPIR-V generator, we know that the value passed to
      // __translate_ocl_memory_scope is what we should pass to the
      // OpenCL builtin now.
      return CI->getArgOperand(0);
    }
  }

  return getOrCreateSwitchFunc(
      kSPIRVName::TranslateSPIRVMemScope, MemScope, OCLMemScopeMap::getRMap(),
      /* IsReverse */ true, std::nullopt, InsertBefore);
}

Value *
SPIRV::transSPIRVMemorySemanticsIntoOCLMemoryOrder(Value *MemorySemantics,
                                                   Instruction *InsertBefore) {
  if (auto *C = dyn_cast<ConstantInt>(MemorySemantics)) {
    return ConstantInt::get(C->getType(),
                            mapSPIRVMemSemanticToOCL(C->getZExtValue()).second);
  }

  if (auto *CI = dyn_cast<CallInst>(MemorySemantics)) {
    Function *F = CI->getCalledFunction();
    if (F && F->getName() == kSPIRVName::TranslateOCLMemOrder) {
      // In case the SPIR-V module was created from an OpenCL program by
      // *this* SPIR-V generator, we know that the value passed to
      // __translate_ocl_memory_order is what we should pass to the
      // OpenCL builtin now.
      return CI->getArgOperand(0);
    }
  }

  // SPIR-V MemorySemantics contains both OCL mem_fence_flags and mem_order and
  // therefore, we need to apply mask
  int Mask = MemorySemanticsMaskNone | MemorySemanticsAcquireMask |
             MemorySemanticsReleaseMask | MemorySemanticsAcquireReleaseMask |
             MemorySemanticsSequentiallyConsistentMask;
  return getOrCreateSwitchFunc(kSPIRVName::TranslateSPIRVMemOrder,
                               MemorySemantics, OCLMemOrderMap::getRMap(),
                               /* IsReverse */ true, std::nullopt, InsertBefore,
                               Mask);
}

Value *SPIRV::transSPIRVMemorySemanticsIntoOCLMemFenceFlags(
    Value *MemorySemantics, Instruction *InsertBefore) {
  if (auto *C = dyn_cast<ConstantInt>(MemorySemantics)) {
    return ConstantInt::get(C->getType(),
                            mapSPIRVMemSemanticToOCL(C->getZExtValue()).first);
  }

  // TODO: any possible optimizations?
  // SPIR-V MemorySemantics contains both OCL mem_fence_flags and mem_order and
  // therefore, we need to apply mask
  int Mask = MemorySemanticsWorkgroupMemoryMask |
             MemorySemanticsCrossWorkgroupMemoryMask |
             MemorySemanticsImageMemoryMask;
  return getOrCreateSwitchFunc(
      kSPIRVName::TranslateSPIRVMemFence, MemorySemantics,
      OCLMemFenceExtendedMap::getRMap(),
      /* IsReverse */ true, std::nullopt, InsertBefore, Mask);
}

void llvm::mangleOpenClBuiltin(const std::string &UniqName,
                               ArrayRef<Type *> ArgTypes,
                               std::string &MangledName) {
  OCLUtil::OCLBuiltinFuncMangleInfo BtnInfo;
  MangledName = SPIRV::mangleBuiltin(UniqName, ArgTypes, &BtnInfo);
}
